00000800:00000200:0.1F:1775763625.319225:0:2377:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000ddc89315] -> 12345-192.168.202.146@tcp (2) 00000800:00000200:1.1F:1775763625.354025:0:2368:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 1, ksnr_connected 2 ksnr_max_conns 1 00000800:00000200:1.1:1775763625.354043:0:2368:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.146@tcp ip 192.168.202.146:988 00000800:00000200:1.1:1775763625.354050:0:2368:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 000000005d5a6c86 type 2, nob 96 niov 1 nkiov 0 00000800:00000200:1.1:1775763625.400279:0:2368:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 2, ksnr_connected 6 ksnr_max_conns 1 00000800:00000200:3.1F:1775763625.408409:0:2368:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 3, ksnr_connected e ksnr_max_conns 1 00000080:00200000:0.0:1775764768.969299:0:19723:0:(rw26.c:916:ll_write_end()) vmpage @000000000434f0cf 17ffffc000107d 2:0 ffff8f9970f91460 0 lru 00000080:00200000:0.0:1775764768.969308:0:19723:0:(rw26.c:916:ll_write_end()) osc-page@00000000243ce695 0: 1< 2 + - > 2< 0 0 4096 0x0 0x40420 | 000000004b957d6e 000000003be2520d > 3< 1 0 > 4< 0 0 8 462946304 - | - - + - > 5< - - + - | 0 - | 1 - -> 00000080:00200000:0.0:1775764768.969324:0:19723:0:(rw26.c:916:ll_write_end()) end page@000000003b9ea4b1 00000080:00200000:0.0:1775764768.969326:0:19723:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764768.969329:0:19723:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000080:00200000:0.0:1775764768.969346:0:19723:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8f9952e2e578, count 1, dirtied 0 00000080:00200000:0.0:1775764768.969351:0:19723:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764768.969358:0:19723:0:(vvp_io.c:1353:vvp_io_write_start()) COURSES.X04: write bytes 0, result: 4096 00000020:00010000:0.0:1775764768.969368:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99713e93f8 (ffff8f9966dc2638/0) at cl_lock_release():245 00000020:00010000:0.0:1775764768.969371:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99713e93f8 (ffff8f9966dc2638/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764768.969375:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0df2b0 (ffff8f9966dc2bd8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764768.969385:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000047343c86/0x36f9bdc9e41cac1a lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x57:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4eea58 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 0 00010000:00010000:0.0:1775764768.969397:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000047343c86/0x36f9bdc9e41cac1a lrc: 4/0,0 mode: PW/PW res: [0x280000403:0x57:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eea58 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 0 00000020:00010000:0.0:1775764768.969411:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99713e93f8 (ffff8f9966dc2638/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764768.969414:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0df2b0 (ffff8f9966dc2638/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764768.969424:0:19723:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764768.969431:0:19723:0:(file.c:2173:ll_file_io_generic()) Range unlock [0, 0] 00000080:00200000:0.0:1775764768.969558:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2c2:0x0] ignore/verify layout 0/0, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764768.969563:0:19723:0:(file.c:2206:ll_file_io_generic()) COURSES.X04: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764768.969574:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2c2:0x0] updating i_size 260096 i_blocks 515 00000080:00200000:0.0:1775764768.969580:0:19723:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764768.969627:0:19723:0:(file.c:2612:do_file_write_iter()) START file COURSES.Y04:[0x200000401:0x2c4:0x0], ppos: 0, count: 4096 00000080:00200000:0.0:1775764768.969666:0:19723:0:(file.c:2058:ll_file_io_generic()) COURSES.Y04: write ppos: 0, bytes: 4096 00000080:00200000:0.0:1775764768.969671:0:19723:0:(file.c:1815:ll_io_set_mirror()) COURSES.Y04: desiginated mirror: 0 00000020:00200000:0.0:1775764768.969677:0:19723:0:(cl_io.c:214:cl_io_rw_init()) header@00000000a800855e[0x110, 65, [0x200000401:0x2c4:0x0] hash] 00000020:00200000:0.0:1775764768.969679:0:19723:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 4096) 0 0 00000080:00200000:0.0:1775764768.969681:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2c4:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764768.969693:0:19723:0:(file.c:2136:ll_file_io_generic()) Range lock [0, 0] 00020000:00200000:0.0:1775764768.969699:0:19723:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 4096) 4096, 4096 00020000:00200000:0.0:1775764768.969702:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764768.969724:0:19723:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:0.0:1775764768.969736:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764768.969866:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0df2b0, osc lock ffff8f996686f430, flags 0x0 00010000:00010000:0.0:1775764768.969889:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000009c24ad6b/0x36f9bdc9e41cac6e lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x58:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eeb23 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 70 lru_type: 1 00010000:00010000:0.0:1775764768.969903:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000009c24ad6b/0x36f9bdc9e41cac6e lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x58:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eeb23 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 70 lru_type: 1 00010000:00010000:0.0:1775764768.969914:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000009c24ad6b/0x36f9bdc9e41cac6e lrc: 6/0,2 mode: PW/PW res: [0x280000403:0x58:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4eeb23 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 70 lru_type: 1 00010000:00010000:0.0:1775764768.969922:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000009c24ad6b/0x36f9bdc9e41cac6e lrc: 6/0,2 mode: PW/PW res: [0x280000403:0x58:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4eeb23 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 70 lru_type: 1 00010000:00010000:0.0:1775764768.969929:0:19723:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000009c24ad6b/0x36f9bdc9e41cac6e lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x58:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4eeb23 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 70 lru_type: 1 00000080:00200000:0.0:1775764768.969938:0:19723:0:(vvp_io.c:1286:vvp_io_write_start()) COURSES.Y04: write [0, 4096) append 0 append lockpos 0 00000080:00200000:0.0:1775764768.969950:0:19723:0:(rw26.c:663:ll_write_begin()) Writing 0 of 0 to 4096 bytes 00000080:00200000:0.0:1775764768.969959:0:19723:0:(rw26.c:858:ll_write_end()) pos 0, len 4096, copied 4096 00000080:00200000:0.0:1775764768.969963:0:19723:0:(rw26.c:916:ll_write_end()) page@0000000098713a96[3 000000001b569f15 2 1 00000000195217ae] 00000080:00200000:0.0:1775764768.969966:0:19723:0:(rw26.c:916:ll_write_end()) vmpage @00000000132772c1 17ffffc000106d 2:0 ffff8f9970b99b60 0 lru 00000080:00200000:0.0:1775764768.969972:0:19723:0:(rw26.c:916:ll_write_end()) osc-page@00000000e9d72131 0: 1< 2 - - > 2< 0 0 4096 0x0 0x40420 | 000000004b957d6e 0000000074dcfdb7 > 3< 0 0 > 4< 0 0 8 462946304 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1775764768.969974:0:19723:0:(rw26.c:916:ll_write_end()) end page@0000000098713a96 00000080:00200000:0.0:1775764768.969975:0:19723:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764768.969978:0:19723:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764768.969994:0:19723:0:(osc_cache.c:848:osc_extent_find()) ### extent: 00000000784ef908 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000009c24ad6b/0x36f9bdc9e41cac6e lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x58:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eeb23 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 70 lru_type: 1 00000080:00200000:0.0:1775764768.970012:0:19723:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8f9952e2c478, count 1, dirtied 1 00000080:00200000:0.0:1775764768.970015:0:19723:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764768.970017:0:19723:0:(vvp_io.c:1353:vvp_io_write_start()) COURSES.Y04: write bytes 0, result: 4096 00000020:00010000:0.0:1775764768.970029:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99713e93f8 (ffff8f9966dc2638/0) at cl_lock_release():245 00000020:00010000:0.0:1775764768.970032:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99713e93f8 (ffff8f9966dc2638/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764768.970035:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0df2b0 (ffff8f9966dc2bd8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764768.970038:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000009c24ad6b/0x36f9bdc9e41cac6e lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x58:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4eeb23 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 70 lru_type: 1 00010000:00010000:0.0:1775764768.970046:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000009c24ad6b/0x36f9bdc9e41cac6e lrc: 4/0,0 mode: PW/PW res: [0x280000403:0x58:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eeb23 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 71 lru_type: 1 00000020:00010000:0.0:1775764768.970056:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99713e93f8 (ffff8f9966dc2638/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764768.970058:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0df2b0 (ffff8f9966dc2638/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764768.970063:0:19723:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764768.970066:0:19723:0:(file.c:2173:ll_file_io_generic()) Range unlock [0, 0] 00000080:00200000:0.0:1775764768.970078:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2c4:0x0] ignore/verify layout 0/0, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764768.970081:0:19723:0:(file.c:2206:ll_file_io_generic()) COURSES.Y04: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764768.970086:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2c4:0x0] updating i_size 260096 i_blocks 515 00000080:00200000:0.0:1775764768.970089:0:19723:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200000:0.0:1775764768.970126:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764768.970132:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764768.970138:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:0.0:1775764768.970144:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764768.970147:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:0.0:1775764768.970150:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764768.970153:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:0.0:1775764768.970156:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764768.970159:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:0.0:1775764768.970161:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764768.970177:0:19723:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=STUDENTS, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), flags=1 00000080:00200000:0.0:1775764768.970181:0:19723:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=STUDENTS, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), intent=getattr 00000002:00010000:0.0:1775764768.970204:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: STUDENTS,[0x0:0x0:0x0]) in obj [0x200000401:0x2bb:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764768.970250:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421be15 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764768.970258:0:19723:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421be15 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764768.970268:0:19723:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421be15 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764768.970290:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4380 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966862592:192.168.202.146@tcp:101:dbench.0 00000100:00100000:0.0:1775764768.970388:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16fd80 going to sleep for 11 seconds 00000100:00100000:0.0:1775764768.986090:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4380 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966862592:192.168.202.146@tcp:101:dbench.0 00010000:00010000:0.0:1775764768.986111:0:19723:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421be15 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764768.986124:0:19723:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421be15 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x206400000000 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764768.986132:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421be15 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x10206400000000 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764768.986140:0:19723:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421be15 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x10206400000000 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764768.986149:0:19723:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421be15 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x206400000000 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764768.986155:0:19723:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99435d5a00 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:0.0:1775764768.986159:0:19723:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421be15 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x2/0x2 rrc: 5 type: IBT flags: 0x206400000000 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764768.986166:0:19723:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421be15 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x2/0x2 rrc: 5 type: IBT flags: 0x206400000000 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764768.986172:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421be15 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x2/0x2 rrc: 5 type: IBT flags: 0x20e400000000 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764768.986179:0:19723:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1775764768.986186:0:19723:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421be15 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x2/0x2 rrc: 5 type: IBT flags: 0x4a0f400000000 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764768.986192:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421be15 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x2/0x2 rrc: 5 type: IBT flags: 0x4a0f400000000 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764768.986210:0:19723:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=7, status=0 req@ffff8f994f2e4380 x1862022966862592/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/584 e 0 to 0 dl 1775764784 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'dbench.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1775764768.986457:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2bb:0x0] suppgids 0 -1: rc 0 00010000:00010000:0.0:1775764768.986485:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c600/0x36f9bdc9e421bcf6 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7b71 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:0.0:1775764768.986499:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c600/0x36f9bdc9e421bcf6 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7b71 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00010000:0.0:1775764768.986513:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421bcf6 from it ffffafbac1c7b930 00010000:00010000:0.0:1775764768.986517:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c600/0x36f9bdc9e421bcf6 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f7b71 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:0.0:1775764768.986526:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c600/0x36f9bdc9e421bcf6 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7b71 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00000080:00200000:0.0:1775764768.989030:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764768.989039:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764768.989046:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=19 00000080:00200000:0.0:1775764768.989053:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764768.989057:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=19 00000080:00200000:0.0:1775764768.989059:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764768.989063:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=19 00000080:00200000:0.0:1775764768.989066:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764768.989070:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=259 00000080:00200000:0.0:1775764768.989074:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 44 00000080:00200008:0.0:1775764768.989082:0:19723:0:(file.c:1057:ll_file_open()) START file PARADOX:[0x200000401:0x2bb:0x0](ffff8f9952da4b08), flags 304000 00000080:00200008:0.0:1775764768.989102:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file PARADOX:[0x200000401:0x2bb:0x0](ffff8f9952da4b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764768.989108:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08),name=PARADOX 00000002:00010000:0.0:1775764768.989128:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2bb:0x0]) in obj [0x200000401:0x26c:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764768.989145:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c600/0x36f9bdc9e421bcf6 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7b71 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00010000:00010000:0.0:1775764768.989157:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c600/0x36f9bdc9e421bcf6 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7b71 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00000080:00200000:0.0:1775764768.989170:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26c:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764768.989175:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2bb:0x0](ffff8f9952da4b08) 00000080:00010000:0.0:1775764768.989179:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2bb:0x0](ffff8f9952da4b08) for lock 0x36f9bdc9e421bcf6 00000080:00010000:0.0:1775764768.989184:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421bcf6 from it ffffafbac1c7bcd8 00010000:00010000:0.0:1775764768.989187:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c600/0x36f9bdc9e421bcf6 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f7b71 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00010000:00010000:0.0:1775764768.989195:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c600/0x36f9bdc9e421bcf6 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7b71 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00000080:00200000:0.0:1775764768.990095:0:19723:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08) pos/size0/8192 32bit_api 0 00000080:00200000:0.0:1775764768.990133:0:19723:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08) pos/size18446744073709551614/8192 32bit_api 0 00000080:00200008:0.0:1775764768.990149:0:19723:0:(file.c:447:ll_file_release()) START file PARADOX:[0x200000401:0x2bb:0x0](ffff8f9952da4b08), flags 304000 00010000:00010000:0.0:1775764768.990165:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d9c00/0x36f9bdc9e41cab4f lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ee860 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764768.990762:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764768.990769:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764768.990774:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:0.0:1775764768.990779:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764768.990782:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:0.0:1775764768.990784:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764768.990787:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:0.0:1775764768.990789:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764768.990792:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:0.0:1775764768.990793:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764768.990809:0:19723:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=STUDENTS.DB, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), flags=1 00000080:00200000:0.0:1775764768.990813:0:19723:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=STUDENTS.DB, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), intent=getattr 00000002:00010000:0.0:1775764768.990828:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: STUDENTS.DB,[0x0:0x0:0x0]) in obj [0x200000401:0x2bb:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764768.990880:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5800/0x36f9bdc9e421be1c lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764768.990887:0:19723:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5800/0x36f9bdc9e421be1c lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764768.990895:0:19723:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5800/0x36f9bdc9e421be1c lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764768.990919:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4380 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966862720:192.168.202.146@tcp:101:dbench.0 00010000:00010000:0.0:1775764769.024481:0:5572:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1775764769.024492:0:5572:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5800/0x36f9bdc9e421be1c lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f7ee3 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.024505:0:5572:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99435d5800 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:0.0:1775764769.024510:0:5572:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5800/0x36f9bdc9e421be1c lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c6:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f7ee3 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.024519:0:5572:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5800/0x36f9bdc9e421be1c lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c6:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f7ee3 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.024528:0:5572:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5800/0x36f9bdc9e421be1c lrc: 3/0,0 mode: PR/PR res: [0x200000401:0x2c6:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f7ee3 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764769.026402:0:11131:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978346624 00000100:00100000:0.0:1775764769.026410:0:11131:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978346624 00000100:00100000:0.0:1775764769.026473:0:11131:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24017 00000100:00100000:0.0:1775764769.026483:0:11131:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970c01500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:3f83546c-01a9-4156-8a97-44c446245d3e+5:5794:x1862022978346624:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764769.026492:0:11131:0:(ldlm_lockd.c:2448:ldlm_callback_handler()) callback on lock 0x36f9bdc9e421be1c - lock disappeared 00010000:00010000:0.0:1775764769.026523:0:11131:0:(ldlm_lockd.c:2345:ldlm_callback_errmsg()) @@@ Operate with invalid parameter, NID=12345-192.168.202.146@tcp lock=0x36f9bdc9e421be1c: rc = 0 req@ffff8f9970c01500 x1862022978346624/t0(0) o104->3f83546c-01a9-4156-8a97-44c446245d3e@192.168.202.146@tcp:250/0 lens 328/224 e 0 to 0 dl 1775764780 ref 1 fl Interpret:/0/0 rc -22/-22 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764769.026706:0:11131:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970c01500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:3f83546c-01a9-4156-8a97-44c446245d3e+5:5794:x1862022978346624:12345-192.168.202.146@tcp:104: Request processed in 224us (468us total) trans 0 rc -22/-22 00000100:00100000:0.0:1775764769.026714:0:11131:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24017 00000100:00100000:0.0:1775764769.028468:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978346752 00000100:00100000:0.0:1775764769.028476:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978346752 00000100:00100000:0.0:1775764769.028528:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24018 00000100:00100000:0.0:1775764769.028708:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970c02d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+5:5794:x1862022978346752:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764769.028725:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421ba33 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x2aac6a50e5f72cd expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764769.028832:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764769.028839:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970c02d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+5:5794:x1862022978346752:12345-192.168.202.146@tcp:104: Request processed in 135us (1016us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764769.028848:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24018 00000100:00100000:0.0:1775764769.072032:0:11131:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978346880 00000100:00100000:0.0:1775764769.072046:0:11131:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978346880 00000100:00100000:0.0:1775764769.072110:0:11131:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24019 00000100:00100000:0.0:1775764769.072119:0:11131:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970c02a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:3f83546c-01a9-4156-8a97-44c446245d3e+5:5796:x1862022978346880:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764769.072136:0:11131:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970972400/0x36f9bdc9e421be23 lrc: 2/0,0 mode: CW/CW res: [0x200000401:0x2c6:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x2aac6a50e5f7ef1 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764769.072235:0:11131:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764769.072242:0:11131:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970c02a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:3f83546c-01a9-4156-8a97-44c446245d3e+5:5796:x1862022978346880:12345-192.168.202.146@tcp:104: Request processed in 124us (7338us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764769.072249:0:11131:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24019 00000100:00100000:0.0:1775764769.072895:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994eeac000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966864768:192.168.202.146@tcp:35:dbench.0 00000002:00100000:0.0:1775764769.072916:0:19724:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f994f0a9500 x1862022966825088/t4294997205(4294997205) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/952 e 0 to 0 dl 1775764783 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000080:00200008:0.0:1775764769.073001:0:19724:0:(file.c:447:ll_file_release()) START file ENROLL.PX:[0x200000401:0x2d3:0x0](ffff8f9952e2a1c8), flags 100002 00000002:00100000:0.0:1775764769.073032:0:19724:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f994f0a9f80 x1862022966825984/t4294997206(4294997206) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/952 e 0 to 0 dl 1775764783 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1775764769.073042:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1775764769.073046:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e4000 x1862022966865280/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1775764769.073072:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966865280:192.168.202.146@tcp:35:dbench.0 00000100:00100000:0.0:1775764769.073114:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16fd80 going to sleep for 11 seconds 00010000:00010000:0.0:1775764769.092795:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1775764769.092802:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970973200/0x36f9bdc9e421be31 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f7f22 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.092811:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970973200 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:0.0:1775764769.092815:0:5571:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970973200/0x36f9bdc9e421be31 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c6:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f7f22 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.092822:0:5571:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970973200/0x36f9bdc9e421be31 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c6:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f7f22 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.092839:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970973200/0x36f9bdc9e421be31 lrc: 3/0,0 mode: PR/PR res: [0x200000401:0x2c6:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f7f22 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.092855:0:5571:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970972600/0x36f9bdc9e421be2a lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f7f06 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764769.092881:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970973200/0x36f9bdc9e421be31 lrc: 2/0,0 mode: --/PR res: [0x200000401:0x2c6:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f7f22 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.092887:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764769.092905:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970973200/0x36f9bdc9e421be31 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2c6:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f7f22 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.092912:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970973200/0x36f9bdc9e421be31 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2c6:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f7f22 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.092923:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764769.092924:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764769.092925:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764769.092932:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1775764769.092943:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966866304:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764769.095942:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966866304:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764769.120560:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764769.120780:0:11131:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1775764769.120796:0:11131:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978347136 00000100:00100000:0.0:1775764769.120805:0:11131:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978347136 00000100:00100000:0.0:1775764769.120854:0:11131:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24021 00000100:00100000:0.0:1775764769.120864:0:11131:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9966932850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:3f83546c-01a9-4156-8a97-44c446245d3e+5:11866:x1862022978347136:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764769.120882:0:11131:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970972400/0x36f9bdc9e421be38 lrc: 2/0,0 mode: CW/CW res: [0x200000401:0x2c6:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x2aac6a50e5f7f30 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764769.120942:0:11131:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764769.120960:0:11131:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9966932850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:3f83546c-01a9-4156-8a97-44c446245d3e+5:11866:x1862022978347136:12345-192.168.202.146@tcp:104: Request processed in 87us (399us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764769.120967:0:11131:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24021 00010000:00010000:0.0:1775764769.140657:0:5572:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1775764769.140664:0:5572:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970924600/0x36f9bdc9e421be46 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f7f61 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.140674:0:5572:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970924600 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:0.0:1775764769.140678:0:5572:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970924600/0x36f9bdc9e421be46 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c8:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f7f61 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.140687:0:5572:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970924600/0x36f9bdc9e421be46 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c8:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f7f61 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.140695:0:5572:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970924600/0x36f9bdc9e421be46 lrc: 3/0,0 mode: PR/PR res: [0x200000401:0x2c8:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f7f61 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.140712:0:5572:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8ca00/0x36f9bdc9e421b74d lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f6a06 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764769.140759:0:5572:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970924600/0x36f9bdc9e421be46 lrc: 2/0,0 mode: --/PR res: [0x200000401:0x2c8:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f7f61 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.140772:0:5572:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764769.140794:0:5572:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970924600/0x36f9bdc9e421be46 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2c8:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f7f61 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.140803:0:5572:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970924600/0x36f9bdc9e421be46 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2c8:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f7f61 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.140817:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764769.140820:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764769.140823:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764769.140836:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966868480:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764769.173048:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966868480:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764769.174066:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764769.189006:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000080:00200000:0.0:1775764769.194911:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764769.194920:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764769.194927:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:0.0:1775764769.194934:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764769.194938:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:0.0:1775764769.194941:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764769.194945:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:0.0:1775764769.194948:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764769.194951:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:0.0:1775764769.194954:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764769.194970:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=__50172.DB, flags=1 00000080:00200008:0.0:1775764769.194978:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file __50172.DB:[0x200000401:0xed5:0x0](ffff8f9952e4aa08), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764769.194984:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xed5:0x0](ffff8f9952e4aa08),name=__50172.DB 00000002:00010000:0.0:1775764769.195005:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xed5:0x0]) in obj [0x200000401:0x2b8:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764769.195023:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925400/0x36f9bdc9e421bde4 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xed5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f7e2d expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 8 lru_type: 1 00010000:00010000:0.0:1775764769.195037:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925400/0x36f9bdc9e421bde4 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xed5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f7e2d expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 8 lru_type: 1 00000080:00200000:0.0:1775764769.195046:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2b8:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764769.195052:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xed5:0x0](ffff8f9952e4aa08) 00000080:00010000:0.0:1775764769.195055:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xed5:0x0](ffff8f9952e4aa08) for lock 0x36f9bdc9e421bde4 00000080:00010000:0.0:1775764769.195061:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421bde4 from it ffffafbac1c5bcb8 00010000:00010000:0.0:1775764769.195066:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925400/0x36f9bdc9e421bde4 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xed5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f7e2d expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 8 lru_type: 1 00010000:00010000:0.0:1775764769.195075:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925400/0x36f9bdc9e421bde4 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xed5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f7e2d expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 9 lru_type: 1 00000080:00200000:0.0:1775764769.195124:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xed5:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1775764769.195136:0:19724:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xed5:0x0] 00000008:00010000:0.0:1775764769.195169:0:19724:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1775764769.195172:0:19724:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0df2b0, osc lock ffff8f996686f430, flags 0x400001002 00010000:00010000:0.0:1775764769.195191:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000121a9b09/0x36f9bdc9e421bcb7 lrc: 3/0,1 mode: PW/PW res: [0x240000403:0x65f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f7a91 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 1 00010000:00010000:0.0:1775764769.195202:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000121a9b09/0x36f9bdc9e421bcb7 lrc: 3/0,1 mode: PW/PW res: [0x240000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f7a91 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 1 00000020:00200000:0.0:1775764769.195220:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1775764769.195224:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000121a9b09/0x36f9bdc9e421bcb7 lrc: 6/0,1 mode: PW/PW res: [0x240000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5f7a91 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 1 00010000:00010000:0.0:1775764769.195231:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000121a9b09/0x36f9bdc9e421bcb7 lrc: 5/0,0 mode: PW/PW res: [0x240000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f7a91 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 1 00000020:00200000:0.0:1775764769.195242:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1775764769.195251:0:19724:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0xed5:0x0] updating i_size 4096 i_blocks 1 00000020:00010000:0.0:1775764769.195255:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970b8ab58 (ffff8f9966dc2638/0) at cl_lock_release():245 00000020:00010000:0.0:1775764769.195258:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970b8ab58 (ffff8f9966dc2638/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764769.195261:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0df2b0 (ffff8f9966dc2bd8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764769.195265:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970b8ab58 (ffff8f9966dc2638/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764769.195268:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0df2b0 (ffff8f9966dc2638/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764769.195292:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xed5:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764769.195314:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764769.195318:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764769.195321:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:0.0:1775764769.195325:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764769.195328:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=16 00000080:00200000:0.0:1775764769.195330:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764769.195332:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:0.0:1775764769.195335:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764769.195337:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=16 00000080:00200000:0.0:1775764769.195339:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764769.195343:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=__50172.DB, flags=0 00000080:00200000:0.0:1775764769.195347:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 3 00000080:00200000:0.0:1775764769.195351:0:19724:0:(namei.c:2395:ll_unlink()) VFS Op:name=__50172.DB, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8) 00010000:00010000:0.0:1775764769.195388:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925c00/0x36f9bdc9e421bd58 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f7c90 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:0.0:1775764769.211116:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e41caad1 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x5/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ee710 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764769.211145:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970abee00/0x36f9bdc9e41cab09 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x11/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ee795 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 77 lru_type: 1 00010000:00010000:0.0:1775764769.211227:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925400/0x36f9bdc9e421bde4 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xed5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f7e2d expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 9 lru_type: 1 00000080:00200000:0.0:1775764769.211273:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0xed5:0x0]: losing layout lock 00010000:00010000:0.0:1775764769.211328:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925400/0x36f9bdc9e421bde4 lrc: 1/0,0 mode: --/PR res: [0x200000401:0xed5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f7e2d expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 9 lru_type: 1 00010000:00010000:0.0:1775764769.211336:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925c00/0x36f9bdc9e421bd58 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f7c90 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:0.0:1775764769.211342:0:19724:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1775764769.211345:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925400/0x36f9bdc9e421bde4 lrc: 0/0,0 mode: --/PR res: [0x200000401:0xed5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f7e2d expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 9 lru_type: 1 00010000:00010000:0.0:1775764769.211360:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925c00/0x36f9bdc9e421bd58 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f7c90 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000020:00100000:0.0:1775764769.211374:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1775764769.211386:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e4700 x1862022966870784/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764769.211407:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966870784:192.168.202.146@tcp:36:dbench.0 00000100:00100000:0.0:1775764769.211484:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f780 going to sleep for 11 seconds 00010000:00010000:0.0:1775764769.215113:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1775764769.215120:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970973200/0x36f9bdc9e421be5b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ca:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f7fa0 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.215128:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970973200 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:0.0:1775764769.215132:0:5571:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970973200/0x36f9bdc9e421be5b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ca:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f7fa0 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.215141:0:5571:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970973200/0x36f9bdc9e421be5b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ca:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f7fa0 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.215148:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970973200/0x36f9bdc9e421be5b lrc: 3/0,0 mode: PR/PR res: [0x200000401:0x2ca:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f7fa0 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.215163:0:5571:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966c09400/0x36f9bdc9e421b762 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ca:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f6a45 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764769.215190:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970973200/0x36f9bdc9e421be5b lrc: 2/0,0 mode: --/PR res: [0x200000401:0x2ca:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f7fa0 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.215197:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764769.215229:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970973200/0x36f9bdc9e421be5b lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2ca:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f7fa0 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.215236:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970973200/0x36f9bdc9e421be5b lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2ca:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f7fa0 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.215249:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764769.215251:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764769.215253:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764769.215262:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1775764769.215280:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966871040:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764769.218256:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966870784:192.168.202.146@tcp:36:dbench.0 00000080:00200000:0.0:1775764769.218357:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xed5:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764769.218372:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1775764769.218394:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:0.0:1775764769.218396:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000008:00010000:0.0:1775764769.218735:0:19724:0:(osc_cache.c:1167:osc_extent_make_ready()) ### extent: 00000000d715df04 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000121a9b09/0x36f9bdc9e421bcb7 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f7a91 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 1 00000008:00010000:0.0:1775764769.218835:0:19724:0:(osc_cache.c:901:osc_extent_finish()) ### extent: 00000000d715df04 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000121a9b09/0x36f9bdc9e421bcb7 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f7a91 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 1 00000008:00010000:0.0:1775764769.218864:0:19724:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000d715df04 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000121a9b09/0x36f9bdc9e421bcb7 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f7a91 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 1 00000080:00200000:0.0:1775764769.218906:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xed5:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764769.218942:0:19724:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0xed5:0x0](ffff8f9952e4aa08) 00000008:00010000:0.0:1775764769.218981:0:19724:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 4096 blocks 1 [cma]time 1775764768 1775764768 0 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000121a9b09/0x36f9bdc9e421bcb7 lrc: 1/0,0 mode: PW/PW res: [0x240000403:0x65f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f7a91 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 1 00000080:00200000:0.0:1775764769.219320:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764769.219328:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764769.219335:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:0.0:1775764769.219342:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764769.219346:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:0.0:1775764769.219350:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764769.219353:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:0.0:1775764769.219356:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764769.219359:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:0.0:1775764769.219367:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764769.219380:0:19724:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=STUDENTS, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), flags=1 00000080:00200000:0.0:1775764769.219384:0:19724:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=STUDENTS, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), intent=getattr 00000002:00010000:0.0:1775764769.219403:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: STUDENTS,[0x0:0x0:0x0]) in obj [0x200000401:0x2b8:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764769.219455:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421be69 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.219462:0:19724:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421be69 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.219472:0:19724:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421be69 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764769.219493:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966871296:192.168.202.146@tcp:101:dbench.0 00000100:00100000:0.0:1775764769.220109:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f780 going to sleep for 11 seconds 00000100:00100000:0.0:1775764769.220678:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978347520 00000100:00100000:0.0:1775764769.220686:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978347520 00000100:00100000:0.0:1775764769.220732:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978347648 00000100:00100000:0.0:1775764769.220735:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978347648 00000100:00100000:0.0:1775764769.220780:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24024 00000100:00100000:0.0:1775764769.220789:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970c00a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+6:11866:x1862022978347520:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764769.220797:0:5562:0:(ldlm_lockd.c:2448:ldlm_callback_handler()) callback on lock 0x36f9bdc9e421be5b - lock disappeared 00010000:00010000:0.0:1775764769.220840:0:5562:0:(ldlm_lockd.c:2345:ldlm_callback_errmsg()) @@@ Operate with invalid parameter, NID=12345-192.168.202.146@tcp lock=0x36f9bdc9e421be5b: rc = 0 req@ffff8f9970c00a80 x1862022978347520/t0(0) o104->3f83546c-01a9-4156-8a97-44c446245d3e@192.168.202.146@tcp:250/0 lens 328/224 e 0 to 0 dl 1775764780 ref 1 fl Interpret:/0/0 rc -22/-22 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764769.220853:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970c00a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+6:11866:x1862022978347520:12345-192.168.202.146@tcp:104: Request processed in 66us (674us total) trans 0 rc -22/-22 00000100:00100000:0.0:1775764769.220860:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24024 00000100:00100000:0.0:1775764769.220877:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24025 00000100:00100000:0.0:1775764769.220883:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970c00700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+5:11866:x1862022978347648:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764769.220893:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966c09400/0x36f9bdc9e421b762 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ca:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x2aac6a50e5f6a45 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764769.220933:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764769.220939:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970c00700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+5:11866:x1862022978347648:12345-192.168.202.146@tcp:104: Request processed in 56us (676us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764769.220946:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24025 00000100:00100000:0.0:1775764769.221000:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966871040:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764769.225081:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966871296:192.168.202.146@tcp:101:dbench.0 00010000:00010000:0.0:1775764769.225101:0:19724:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421be69 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.225115:0:19724:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421be69 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.225122:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421be69 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.225130:0:19724:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421be69 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.225139:0:19724:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421be69 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.225146:0:19724:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99435d4800 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:0.0:1775764769.225150:0:19724:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421be69 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x2/0x2 rrc: 4 type: IBT flags: 0x206400000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.225157:0:19724:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421be69 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x2/0x2 rrc: 4 type: IBT flags: 0x206400000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.225164:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421be69 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x2/0x2 rrc: 4 type: IBT flags: 0x20e400000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.225172:0:19724:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1775764769.225177:0:19724:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421be69 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x2/0x2 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.225184:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421be69 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x2/0x2 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764769.225200:0:19724:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=7, status=0 req@ffff8f994f2e6a00 x1862022966871296/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/584 e 0 to 0 dl 1775764785 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'dbench.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1775764769.225214:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2b8:0x0] suppgids 0 -1: rc 0 00010000:00010000:0.0:1775764769.225238:0:19724:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/696 (0 0) 00000080:00200000:0.0:1775764769.225312:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764769.225319:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764769.225326:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=19 00000080:00200000:0.0:1775764769.225333:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764769.225337:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=19 00000080:00200000:0.0:1775764769.225340:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764769.225344:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=19 00000080:00200000:0.0:1775764769.225346:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764769.225350:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=259 00000080:00200000:0.0:1775764769.225353:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 44 00000080:00200008:0.0:1775764769.225361:0:19724:0:(file.c:1057:ll_file_open()) START file PARADOX:[0x200000401:0x2b8:0x0](ffff8f9952d880c8), flags 304000 00000080:00200008:0.0:1775764769.225380:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file PARADOX:[0x200000401:0x2b8:0x0](ffff8f9952d880c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764769.225386:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8),name=PARADOX 00000002:00010000:0.0:1775764769.225402:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2b8:0x0]) in obj [0x200000401:0x26b:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764769.225409:0:19724:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/696 (0 0) 00010000:00010000:0.0:1775764769.225544:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421be70 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.225553:0:19724:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421be70 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.225563:0:19724:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421be70 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764769.225584:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966871552:192.168.202.146@tcp:101:dbench.0 00000100:00100000:0.0:1775764769.226137:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f780 going to sleep for 11 seconds 00000100:00100000:0.0:1775764769.239950:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00010000:00010000:0.0:1775764769.250882:0:5572:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1775764769.250888:0:5572:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421be7e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2dd:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f801e expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.250893:0:5572:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99706eae00 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:0.0:1775764769.250897:0:5572:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421be7e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2dd:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f801e expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.250905:0:5572:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421be7e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2dd:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f801e expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.250910:0:5572:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421be7e lrc: 3/0,0 mode: PR/PR res: [0x200000401:0x2dd:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f801e expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.250921:0:5572:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970924400/0x36f9bdc9e421bc4e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2dd:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f792c expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764769.250939:0:5572:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421be7e lrc: 2/0,0 mode: --/PR res: [0x200000401:0x2dd:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f801e expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.250943:0:5572:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764769.250967:0:5572:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421be7e lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2dd:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f801e expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.250974:0:5572:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421be7e lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2dd:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f801e expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.250987:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764769.250989:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764769.250991:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764769.250998:0:2380:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1775764769.251010:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966872576:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764769.257004:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966872576:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764769.257040:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978347904 00000100:00100000:0.0:1775764769.257046:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978347904 00000100:00100000:0.0:1775764769.257128:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24027 00000100:00100000:0.0:1775764769.257137:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970c01c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+5:9373:x1862022978347904:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764769.257148:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970924400/0x36f9bdc9e421bc4e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2dd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x2aac6a50e5f792c expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764769.257205:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:0.0:1775764769.257211:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970c01c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+5:9373:x1862022978347904:12345-192.168.202.146@tcp:104: Request processed in 75us (266us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764769.257218:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24027 00010000:00010000:0.0:1775764769.257277:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1775764769.257283:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970924400/0x36f9bdc9e421bc4e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2dd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x2aac6a50e5f792c expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764769.257290:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970924400 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:0.0:1775764769.257294:0:19801:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970924400/0x36f9bdc9e421bc4e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2dd:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x2aac6a50e5f792c expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764769.257302:0:19801:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970924400/0x36f9bdc9e421bc4e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2dd:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x2aac6a50e5f792c expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764769.257309:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970924400/0x36f9bdc9e421bc4e lrc: 3/0,0 mode: PR/PR res: [0x200000401:0x2dd:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x2aac6a50e5f792c expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:0.0:1775764769.257345:0:19801:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2dd:0x0]: losing layout lock 00010000:00010000:0.0:1775764769.257386:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970924400/0x36f9bdc9e421bc4e lrc: 2/0,0 mode: --/PR res: [0x200000401:0x2dd:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x2aac6a50e5f792c expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764769.257397:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764769.257622:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970924400/0x36f9bdc9e421bc4e lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2dd:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x2aac6a50e5f792c expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764769.257631:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970924400/0x36f9bdc9e421bc4e lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2dd:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x2aac6a50e5f792c expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764769.257676:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1775764769.257679:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925c00/0x36f9bdc9e421be85 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2cc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f803a expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.257685:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970925c00 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:0.0:1775764769.257687:0:19801:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925c00/0x36f9bdc9e421be85 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2cc:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f803a expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.257694:0:19801:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925c00/0x36f9bdc9e421be85 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2cc:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f803a expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.257700:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925c00/0x36f9bdc9e421be85 lrc: 3/0,0 mode: PR/PR res: [0x200000401:0x2cc:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f803a expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.257712:0:19801:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966c09800/0x36f9bdc9e421b777 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2cc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f6a84 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764769.257729:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925c00/0x36f9bdc9e421be85 lrc: 2/0,0 mode: --/PR res: [0x200000401:0x2cc:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f803a expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.257759:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764769.258052:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925c00/0x36f9bdc9e421be85 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2cc:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f803a expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.258059:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925c00/0x36f9bdc9e421be85 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2cc:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f803a expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.258070:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764769.258072:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764769.258074:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1775764769.267209:0:5571:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2cc:0x0]: losing layout lock 00010000:00010000:0.0:1775764769.267260:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966c09800/0x36f9bdc9e421b777 lrc: 2/0,0 mode: --/PR res: [0x200000401:0x2cc:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x2aac6a50e5f6a84 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764769.267266:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764769.267285:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966c09800/0x36f9bdc9e421b777 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2cc:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x2aac6a50e5f6a84 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764769.267290:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966c09800/0x36f9bdc9e421b777 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2cc:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x2aac6a50e5f6a84 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764769.267306:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764769.267307:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764769.267308:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764769.267320:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966873344:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764769.272367:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966873344:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764769.282291:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978348288 00000100:00100000:0.0:1775764769.282299:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978348288 00000100:00100000:0.0:1775764769.282356:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24030 00000100:00100000:0.0:1775764769.282365:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970c00700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+5:5796:x1862022978348288:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764769.282379:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925400/0x36f9bdc9e421be93 lrc: 2/0,0 mode: CW/CW res: [0x200000401:0x2cc:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x2aac6a50e5f805d expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764769.282434:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764769.282440:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970c00700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+5:5796:x1862022978348288:12345-192.168.202.146@tcp:104: Request processed in 77us (234us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764769.282600:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24030 00010000:00010000:0.0:1775764769.285912:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764769.305176:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970c00380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966874880:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764769.316148:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970c00380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966874880:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764769.318388:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978348800 00000100:00100000:0.0:1775764769.318397:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978348800 00000100:00100000:0.0:1775764769.318545:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24034 00000100:00100000:0.0:1775764769.318555:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970c01880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+5:11866:x1862022978348800:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764769.318573:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421beb6 lrc: 2/0,0 mode: CW/CW res: [0x200000401:0x2c1:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x2aac6a50e5f80bf expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764769.318960:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764769.318968:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970c01880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+5:11866:x1862022978348800:12345-192.168.202.146@tcp:104: Request processed in 414us (650us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764769.318976:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24034 00010000:00010000:0.0:1775764769.329456:0:5572:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1775764769.329462:0:5572:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421bed2 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f8113 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.329471:0:5572:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99706eb200 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:0.0:1775764769.329475:0:5572:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421bed2 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d0:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f8113 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.329482:0:5572:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421bed2 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d0:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f8113 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.329490:0:5572:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421bed2 lrc: 3/0,0 mode: PR/PR res: [0x200000401:0x2d0:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f8113 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.329505:0:5572:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966c09e00/0x36f9bdc9e421b7a1 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2d0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f6b02 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764769.329538:0:5572:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421bed2 lrc: 2/0,0 mode: --/PR res: [0x200000401:0x2d0:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8113 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.329545:0:5572:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764769.329578:0:5572:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421bed2 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2d0:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8113 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.329585:0:5572:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421bed2 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2d0:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8113 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.329597:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764769.329599:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764769.329601:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764769.329609:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1775764769.329623:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966875776:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1775764769.334106:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1775764769.334114:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925400/0x36f9bdc9e421bed9 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f812f expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.334122:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970925400 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:0.0:1775764769.334128:0:19801:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925400/0x36f9bdc9e421bed9 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c3:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f812f expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.334136:0:19801:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925400/0x36f9bdc9e421bed9 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c3:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f812f expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.334145:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925400/0x36f9bdc9e421bed9 lrc: 3/0,0 mode: PR/PR res: [0x200000401:0x2c3:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f812f expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.334162:0:19801:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093ce00/0x36f9bdc9e421bae2 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f74d3 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764769.334199:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925400/0x36f9bdc9e421bed9 lrc: 2/0,0 mode: --/PR res: [0x200000401:0x2c3:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f812f expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.334206:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764769.334226:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925400/0x36f9bdc9e421bed9 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2c3:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f812f expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.334233:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925400/0x36f9bdc9e421bed9 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2c3:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f812f expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.334250:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764769.334253:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764769.334254:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1775764769.336803:0:19724:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x2c3:0x0](ffff8f9952e2b248): rc = 0 00000080:00200000:0.0:1775764769.336828:0:19724:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x2c3:0x0]: layout version change: 3 -> 3 00000080:00010000:0.0:1775764769.336839:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2c3:0x0](ffff8f9952e2b248) for lock 0x36f9bdc9e421bae2 00000080:00010000:0.0:1775764769.336851:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2c3:0x0](ffff8f9952e2b248) 00000080:00010000:0.0:1775764769.336854:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421bae2 from it ffffafbac1c5bb20 00010000:00010000:0.0:1775764769.336859:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093ce00/0x36f9bdc9e421bae2 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f74d3 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764769.336871:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093ce00/0x36f9bdc9e421bae2 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f74d3 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200008:0.0:1775764769.336922:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file STUDENTS.PX:[0x200000401:0x2c3:0x0](ffff8f9952e2b248), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764769.336929:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2c3:0x0](ffff8f9952e2b248),name=STUDENTS.PX 00000002:00010000:0.0:1775764769.336949:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2c3:0x0]) in obj [0x200000401:0x2b8:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764769.336964:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093ce00/0x36f9bdc9e421bae2 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2c3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f74d3 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764769.336976:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093ce00/0x36f9bdc9e421bae2 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2c3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f74d3 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764769.336985:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2b8:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764769.336989:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2c3:0x0](ffff8f9952e2b248) 00000080:00010000:0.0:1775764769.338985:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2c3:0x0](ffff8f9952e2b248) for lock 0x36f9bdc9e421bae2 00000080:00010000:0.0:1775764769.339142:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421bae2 from it ffffafbac1c5bcb8 00010000:00010000:0.0:1775764769.339149:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093ce00/0x36f9bdc9e421bae2 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f74d3 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764769.339168:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093ce00/0x36f9bdc9e421bae2 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f74d3 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1775764769.345933:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966876160:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764769.351128:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764769.371813:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764769.371852:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978349312 00000100:00100000:0.0:1775764769.371861:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978349312 00000100:00100000:0.0:1775764769.371908:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24038 00000100:00100000:0.0:1775764769.371918:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e5c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+5:5796:x1862022978349312:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764769.371935:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421bee7 lrc: 2/0,0 mode: CW/CW res: [0x200000401:0x2c3:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x2aac6a50e5f814b expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764769.372009:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764769.372016:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e5c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+5:5796:x1862022978349312:12345-192.168.202.146@tcp:104: Request processed in 100us (212us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764769.372023:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24038 00000100:00100000:0.0:1775764769.389244:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978349568 00000100:00100000:0.0:1775764769.389253:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978349568 00000100:00100000:0.0:1775764769.389304:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24040 00000100:00100000:0.0:1775764769.389313:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970c03b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+5:9373:x1862022978349568:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764769.389326:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970973a00/0x36f9bdc9e421b7d2 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x2aac6a50e5f6b9c expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764769.389383:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764769.389389:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970c03b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+5:9373:x1862022978349568:12345-192.168.202.146@tcp:104: Request processed in 78us (2223us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764769.389396:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24040 00000100:00100000:0.0:1775764769.392012:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764769.392050:0:11131:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978349696 00000100:00100000:0.0:1775764769.392059:0:11131:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978349696 00000100:00100000:0.0:1775764769.392098:0:11131:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24041 00000100:00100000:0.0:1775764769.392103:0:11131:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e7800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:3f83546c-01a9-4156-8a97-44c446245d3e+5:5794:x1862022978349696:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764769.392111:0:11131:0:(ldlm_lockd.c:2448:ldlm_callback_handler()) callback on lock 0x36f9bdc9e421bf03 - lock disappeared 00010000:00010000:0.0:1775764769.392152:0:11131:0:(ldlm_lockd.c:2345:ldlm_callback_errmsg()) @@@ Operate with invalid parameter, NID=12345-192.168.202.146@tcp lock=0x36f9bdc9e421bf03: rc = 0 req@ffff8f994f2e7800 x1862022978349696/t0(0) o104->3f83546c-01a9-4156-8a97-44c446245d3e@192.168.202.146@tcp:250/0 lens 328/224 e 0 to 0 dl 1775764780 ref 1 fl Interpret:/0/0 rc -22/-22 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764769.392160:0:11131:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e7800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:3f83546c-01a9-4156-8a97-44c446245d3e+5:5794:x1862022978349696:12345-192.168.202.146@tcp:104: Request processed in 58us (159us total) trans 0 rc -22/-22 00000100:00100000:0.0:1775764769.392165:0:11131:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24041 00000100:00100000:0.0:1775764769.392195:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764769.392215:0:5561:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1775764769.392224:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978349824 00000100:00100000:0.0:1775764769.392226:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978349824 00000100:00100000:0.0:1775764769.392238:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24042 00000100:00100000:0.0:1775764769.392241:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970e0a450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+5:5794:x1862022978349824:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764769.392250:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093c200/0x36f9bdc9e421baf7 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x2aac6a50e5f7512 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764769.392419:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764769.392425:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970e0a450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+5:5794:x1862022978349824:12345-192.168.202.146@tcp:104: Request processed in 182us (229us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764769.392434:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24042 00000100:00100000:0.0:1775764769.405790:0:11146:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978349952 00000100:00100000:0.0:1775764769.405798:0:11146:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978349952 00000100:00100000:0.0:1775764769.405849:0:11146:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24043 00000100:00100000:0.0:1775764769.405858:0:11146:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970c01880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:3f83546c-01a9-4156-8a97-44c446245d3e+5:5794:x1862022978349952:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764769.405876:0:11146:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970924600/0x36f9bdc9e421bf0a lrc: 2/0,0 mode: CW/CW res: [0x200000401:0x2d2:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x2aac6a50e5f819f expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764769.405942:0:11146:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764769.405948:0:11146:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970c01880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:3f83546c-01a9-4156-8a97-44c446245d3e+5:5794:x1862022978349952:12345-192.168.202.146@tcp:104: Request processed in 92us (328us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764769.405956:0:11146:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24043 00000100:00100000:0.0:1775764769.411375:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000080:00200000:0.0:1775764769.422459:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2d6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1775764769.422471:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2d6:0x0] 00000008:00010000:0.0:1775764769.422502:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1775764769.422505:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0dfe30, osc lock ffff8f996686f430, flags 0x400001002 00010000:00010000:0.0:1775764769.422527:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000260038e6/0x36f9bdc9e41caed6 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x61:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef0e8 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 98 lru_type: 1 00010000:00010000:0.0:1775764769.422540:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000260038e6/0x36f9bdc9e41caed6 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x61:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef0e8 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 98 lru_type: 1 00000020:00200000:0.0:1775764769.422550:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1775764769.422553:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000260038e6/0x36f9bdc9e41caed6 lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x61:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ef0e8 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 98 lru_type: 1 00010000:00010000:0.0:1775764769.422562:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000260038e6/0x36f9bdc9e41caed6 lrc: 4/0,0 mode: PW/PW res: [0x280000403:0x61:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef0e8 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 99 lru_type: 1 00000020:00200000:0.0:1775764769.422570:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1775764769.422577:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2d6:0x0] updating i_size 260096 i_blocks 1 00000020:00010000:0.0:1775764769.422581:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970b8ab58 (ffff8f9966dc2638/0) at cl_lock_release():245 00000020:00010000:0.0:1775764769.422582:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970b8ab58 (ffff8f9966dc2638/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764769.422584:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0dfe30 (ffff8f9966dc2bd8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764769.422587:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970b8ab58 (ffff8f9966dc2638/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764769.422590:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0dfe30 (ffff8f9966dc2638/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764769.422932:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2d6:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764769.423149:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764769.423155:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764769.423162:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:0.0:1775764769.423168:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764769.423172:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:0.0:1775764769.423174:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764769.423177:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:0.0:1775764769.423180:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764769.423183:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:0.0:1775764769.423186:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764769.423189:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=ENROLL.DB, flags=257 00000080:00200000:0.0:1775764769.423193:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2d6:0x0](ffff8f9952d96c08), inode mode 8180 mask 46 00000080:00200008:0.0:1775764769.423200:0:19723:0:(file.c:1057:ll_file_open()) START file ENROLL.DB:[0x200000401:0x2d6:0x0](ffff8f9952d96c08), flags 100002 00000002:00010000:0.0:1775764769.423217:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2d6:0x0]) in obj [0x200000401:0x2bb:0x0], intent: open flags 04240100003 00010000:00010000:0.0:1775764769.423257:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421bf2d lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.423263:0:19723:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421bf2d lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x1/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1775764769.423270:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1775764769.423277:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e5c00 x1862022966879488/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/135824 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1775764769.423284:0:19723:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421bf2d lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x1/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764769.423297:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5c00 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966879488:192.168.202.146@tcp:101:dbench.0 00000100:00100000:0.0:1775764769.424064:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f3c0 going to sleep for 11 seconds 00000100:00100000:0.0:1775764769.451798:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978350336 00000100:00100000:0.0:1775764769.451808:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978350336 00000100:00100000:0.0:1775764769.451867:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24046 00000100:00100000:0.0:1775764769.451878:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970c00700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+5:9373:x1862022978350336:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764769.451894:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421bb0c lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c7:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x2aac6a50e5f7566 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764769.451967:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764769.451974:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970c00700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+5:9373:x1862022978350336:12345-192.168.202.146@tcp:104: Request processed in 99us (289us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764769.451982:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24046 00000100:00100000:0.0:1775764769.467521:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764769.474000:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978350592 00000100:00100000:0.0:1775764769.474011:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978350592 00000100:00100000:0.0:1775764769.474074:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24048 00000100:00100000:0.0:1775764769.474086:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970c03100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+5:5795:x1862022978350592:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764769.474104:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421bf3b lrc: 2/0,0 mode: CW/CW res: [0x200000401:0x2c7:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x2aac6a50e5f8247 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764769.474183:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764769.474191:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970c03100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+5:5795:x1862022978350592:12345-192.168.202.146@tcp:104: Request processed in 107us (452us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764769.474199:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24048 00000001:00010000:0.0:1775764769.492200:0:19724:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2c7:0x0] 00000008:00010000:0.0:1775764769.492255:0:19724:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1775764769.492260:0:19724:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0dfe30, osc lock ffff8f996686f430, flags 0x400001002 00010000:00010000:0.0:1775764769.492286:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001fc9e7d4/0x36f9bdc9e41cacc9 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x5a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eebfc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 45 lru_type: 1 00010000:00010000:0.0:1775764769.492309:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001fc9e7d4/0x36f9bdc9e41cacc9 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x5a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eebfc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 45 lru_type: 1 00000020:00200000:0.0:1775764769.492331:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1775764769.492338:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001fc9e7d4/0x36f9bdc9e41cacc9 lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x5a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4eebfc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 45 lru_type: 1 00010000:00010000:0.0:1775764769.492351:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001fc9e7d4/0x36f9bdc9e41cacc9 lrc: 4/0,0 mode: PW/PW res: [0x240000403:0x5a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eebfc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 46 lru_type: 1 00000020:00200000:0.0:1775764769.492378:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1775764769.492391:0:19724:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2c7:0x0] updating i_size 260096 i_blocks 1 00000020:00010000:0.0:1775764769.492697:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef516d0 (ffff8f9970314998/0) at cl_lock_release():245 00000020:00010000:0.0:1775764769.492701:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef516d0 (ffff8f9970314998/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764769.492705:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0dfe30 (ffff8f9966dc2638/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764769.492709:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef516d0 (ffff8f9970314998/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764769.492712:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0dfe30 (ffff8f9970314998/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764769.492780:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2c7:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:0.0:1775764769.492965:0:19724:0:(file.c:2454:do_file_read_iter()) START file STUDENTS.YG0:[0x200000401:0x2c7:0x0], ppos: 0, count: 4096 00000080:00200000:0.0:1775764769.493011:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764769.493018:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764769.493024:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:0.0:1775764769.493030:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764769.493035:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:0.0:1775764769.493039:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764769.493043:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:0.0:1775764769.493047:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764769.493053:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:0.0:1775764769.493058:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764769.493072:0:19724:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=STUDENTS.XG1, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), flags=1 00000080:00200000:0.0:1775764769.493079:0:19724:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=STUDENTS.XG1, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), intent=getattr 00000002:00010000:0.0:1775764769.493099:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: STUDENTS.XG1,[0x0:0x0:0x0]) in obj [0x200000401:0x2b8:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764769.493134:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421bf57 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.493146:0:19724:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421bf57 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.493158:0:19724:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421bf57 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764769.493179:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7100 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966881536:192.168.202.146@tcp:101:dbench.0 00000100:00100000:0.0:1775764769.493230:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f900 going to sleep for 11 seconds 00000100:00100000:0.0:1775764769.496791:0:11146:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1775764769.496808:0:11146:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978350720 00000100:00100000:0.0:1775764769.496813:0:11146:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978350720 00000100:00100000:0.0:1775764769.496867:0:11146:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24049 00000100:00100000:0.0:1775764769.496875:0:11146:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9966933450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:3f83546c-01a9-4156-8a97-44c446245d3e+5:5796:x1862022978350720:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764769.496902:0:11146:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be200/0x36f9bdc9e421b6ba lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x2aac6a50e5f683f expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764769.496975:0:11146:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764769.496982:0:11146:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9966933450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:3f83546c-01a9-4156-8a97-44c446245d3e+5:5796:x1862022978350720:12345-192.168.202.146@tcp:104: Request processed in 107us (693us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764769.496989:0:11146:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24049 00000100:00100000:0.0:1775764769.509834:0:11131:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978350848 00000100:00100000:0.0:1775764769.509843:0:11131:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978350848 00000100:00100000:0.0:1775764769.509902:0:11131:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24050 00000100:00100000:0.0:1775764769.509911:0:11131:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970c03b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:3f83546c-01a9-4156-8a97-44c446245d3e+5:9373:x1862022978350848:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764769.509928:0:11131:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebe00/0x36f9bdc9e421bf5e lrc: 2/0,0 mode: CW/CW res: [0x200000401:0x2d8:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x2aac6a50e5f829b expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764769.510194:0:11131:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764769.510201:0:11131:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970c03b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:3f83546c-01a9-4156-8a97-44c446245d3e+5:9373:x1862022978350848:12345-192.168.202.146@tcp:104: Request processed in 291us (615us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764769.510209:0:11131:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24050 00010000:00010000:0.0:1775764769.510230:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1775764769.510236:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebe00/0x36f9bdc9e421bf5e lrc: 2/0,0 mode: CW/CW res: [0x200000401:0x2d8:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x2aac6a50e5f829b expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764769.510251:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99706ebe00 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:0.0:1775764769.510256:0:5571:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebe00/0x36f9bdc9e421bf5e lrc: 2/0,0 mode: CW/CW res: [0x200000401:0x2d8:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x2aac6a50e5f829b expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764769.510265:0:5571:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebe00/0x36f9bdc9e421bf5e lrc: 2/0,0 mode: CW/CW res: [0x200000401:0x2d8:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x2aac6a50e5f829b expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764769.510275:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebe00/0x36f9bdc9e421bf5e lrc: 3/0,0 mode: CW/CW res: [0x200000401:0x2d8:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x408400000000 nid: local remote: 0x2aac6a50e5f829b expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764769.510327:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebe00/0x36f9bdc9e421bf5e lrc: 2/0,0 mode: --/CW res: [0x200000401:0x2d8:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x2aac6a50e5f829b expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764769.510334:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764769.510350:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebe00/0x36f9bdc9e421bf5e lrc: 1/0,0 mode: --/CW res: [0x200000401:0x2d8:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x2aac6a50e5f829b expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764769.510358:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebe00/0x36f9bdc9e421bf5e lrc: 0/0,0 mode: --/CW res: [0x200000401:0x2d8:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x2aac6a50e5f829b expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764769.510370:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764769.510372:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764769.510374:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764769.515327:0:11146:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978350976 00000100:00100000:0.0:1775764769.515338:0:11146:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978350976 00000100:00100000:0.0:1775764769.515394:0:11146:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24051 00000100:00100000:0.0:1775764769.515412:0:11146:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970c00a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:3f83546c-01a9-4156-8a97-44c446245d3e+5:5795:x1862022978350976:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764769.515435:0:11146:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093c600/0x36f9bdc9e421bb21 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x2aac6a50e5f75a5 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764769.515514:0:11146:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764769.515521:0:11146:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970c00a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:3f83546c-01a9-4156-8a97-44c446245d3e+5:5795:x1862022978350976:12345-192.168.202.146@tcp:104: Request processed in 111us (288us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764769.515528:0:11146:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24051 00000100:00100000:0.0:1775764769.537136:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764769.537173:0:11131:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978351104 00000100:00100000:0.0:1775764769.537182:0:11131:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978351104 00000100:00100000:0.0:1775764769.537230:0:11131:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24052 00000100:00100000:0.0:1775764769.537237:0:11131:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e7800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:3f83546c-01a9-4156-8a97-44c446245d3e+5:16259:x1862022978351104:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764769.537254:0:11131:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421bf6c lrc: 2/0,0 mode: CW/CW res: [0x200000401:0x2c9:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x2aac6a50e5f82da expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764769.537314:0:11131:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764769.537320:0:11131:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e7800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:3f83546c-01a9-4156-8a97-44c446245d3e+5:16259:x1862022978351104:12345-192.168.202.146@tcp:104: Request processed in 83us (195us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764769.537326:0:11131:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24052 00000080:00200000:0.0:1775764769.583693:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764769.583703:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764769.583708:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:0.0:1775764769.583715:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764769.583719:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:0.0:1775764769.583722:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764769.583725:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:0.0:1775764769.583727:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764769.583731:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:0.0:1775764769.583734:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764769.583755:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=STUDENTS.YG1, flags=257 00000080:00200000:0.0:1775764769.583760:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2cb:0x0](ffff8f9952e2cb08), inode mode 8180 mask 46 00000080:00200008:0.0:1775764769.583770:0:19724:0:(file.c:1057:ll_file_open()) START file STUDENTS.YG1:[0x200000401:0x2cb:0x0](ffff8f9952e2cb08), flags 100002 00000002:00010000:0.0:1775764769.583792:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2cb:0x0]) in obj [0x200000401:0x2b8:0x0], intent: open flags 04240100003 00010000:00010000:0.0:1775764769.583852:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421bf8f lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.583861:0:19724:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421bf8f lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x1/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1775764769.583871:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 2 is allocated opc 101, max 7 00000100:00100000:0.0:1775764769.583883:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e7b80 x1862022966884096/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/135824 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1775764769.583892:0:19724:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421bf8f lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x1/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764769.583912:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966884096:192.168.202.146@tcp:101:dbench.0 00000100:00100000:0.0:1775764769.584105:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16fd80 going to sleep for 11 seconds 00000100:00100000:0.0:1775764769.590453:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978351488 00000100:00100000:0.0:1775764769.590463:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978351488 00000100:00100000:0.0:1775764769.590535:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24055 00000100:00100000:0.0:1775764769.590545:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970c02a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+5:5796:x1862022978351488:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764769.590564:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093c000/0x36f9bdc9e421bb36 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2cb:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x2aac6a50e5f75e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764769.590667:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764769.590674:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970c02a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+5:5796:x1862022978351488:12345-192.168.202.146@tcp:104: Request processed in 131us (347us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764769.590682:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24055 00000100:00100000:0.0:1775764769.600266:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764769.600312:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764769.621867:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970c03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966885376:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764769.623863:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970c03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966885376:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764769.623974:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764769.624000:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978351872 00000100:00100000:0.0:1775764769.624008:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978351872 00000100:00100000:0.0:1775764769.624205:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24057 00000100:00100000:0.0:1775764769.624216:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e5180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+5:5796:x1862022978351872:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764769.624227:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480c00/0x36f9bdc9e421bb4b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2cd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x2aac6a50e5f7623 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764769.624283:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764769.624290:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e5180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+5:5796:x1862022978351872:12345-192.168.202.146@tcp:104: Request processed in 76us (320us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764769.624302:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24057 00000100:00100000:0.0:1775764769.638663:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978352128 00000100:00100000:0.0:1775764769.638671:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978352128 00000100:00100000:0.0:1775764769.638723:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24059 00000100:00100000:0.0:1775764769.638732:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970c02d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+5:5795:x1862022978352128:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764769.638869:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d48200/0x36f9bdc9e421bfb2 lrc: 2/0,0 mode: CW/CW res: [0x200000401:0x2cd:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x2aac6a50e5f83b3 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764769.639018:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764769.639035:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970c02d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+5:5795:x1862022978352128:12345-192.168.202.146@tcp:104: Request processed in 295us (809us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764769.639042:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24059 00000100:00100000:0.0:1775764769.676180:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764769.676207:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764769.676231:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978352384 00000100:00100000:0.0:1775764769.676241:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978352384 00000100:00100000:0.0:1775764769.676282:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978352512 00000100:00100000:0.0:1775764769.676285:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978352512 00000100:00100000:0.0:1775764769.676304:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24061 00000100:00100000:0.0:1775764769.676314:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e7800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+6:16259:x1862022978352384:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764769.676332:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49e00/0x36f9bdc9e421bfce lrc: 3/0,0 mode: PR/PR res: [0x200000401:0x2d1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x2aac6a50e5f8407 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.693238:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/0 locks to priority blp list, 0 blwis in pool 00000100:00100000:0.0:1775764769.693256:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e7800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+5:16259:x1862022978352384:12345-192.168.202.146@tcp:104: Request processed in 16939us (17080us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764769.693268:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24061 00000100:00100000:0.0:1775764769.699475:0:5561:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1775764769.699493:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978352640 00000100:00100000:0.0:1775764769.699501:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978352640 00000100:00100000:0.0:1775764769.699557:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24063 00000100:00100000:0.0:1775764769.699566:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970e0b450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+5:11866:x1862022978352640:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764769.699585:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d48200/0x36f9bdc9e421bfd5 lrc: 2/0,0 mode: CW/CW res: [0x200000401:0x2d1:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x2aac6a50e5f8415 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764769.699662:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764769.699669:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970e0b450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+5:11866:x1862022978352640:12345-192.168.202.146@tcp:104: Request processed in 104us (382us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764769.699676:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24063 00000100:00100000:0.0:1775764769.719254:0:11146:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978353024 00000100:00100000:0.0:1775764769.719264:0:11146:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978353024 00000100:00100000:0.0:1775764769.719320:0:11146:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24064 00000100:00100000:0.0:1775764769.719330:0:11146:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970c00a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:3f83546c-01a9-4156-8a97-44c446245d3e+5:16259:x1862022978353024:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764769.719346:0:11146:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970924400/0x36f9bdc9e421bfea lrc: 2/0,0 mode: CW/CW res: [0x200000401:0x2de:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x2aac6a50e5f8454 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764769.719433:0:11146:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764769.719439:0:11146:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970c00a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:3f83546c-01a9-4156-8a97-44c446245d3e+5:16259:x1862022978353024:12345-192.168.202.146@tcp:104: Request processed in 111us (299us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764769.719446:0:11146:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24064 00000100:00100000:0.0:1775764769.738313:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978353152 00000100:00100000:0.0:1775764769.738323:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978353152 00000100:00100000:0.0:1775764769.738373:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978353280 00000100:00100000:0.0:1775764769.738377:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978353280 00000100:00100000:0.0:1775764769.738410:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24065 00000100:00100000:0.0:1775764769.738419:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970c00380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+6:5795:x1862022978353152:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764769.738430:0:5561:0:(ldlm_lockd.c:2448:ldlm_callback_handler()) callback on lock 0x36f9bdc9e421bff8 - lock disappeared 00010000:00010000:0.0:1775764769.738486:0:5561:0:(ldlm_lockd.c:2345:ldlm_callback_errmsg()) @@@ Operate with invalid parameter, NID=12345-192.168.202.146@tcp lock=0x36f9bdc9e421bff8: rc = 0 req@ffff8f9970c00380 x1862022978353152/t0(0) o104->3f83546c-01a9-4156-8a97-44c446245d3e@192.168.202.146@tcp:250/0 lens 328/224 e 0 to 0 dl 1775764780 ref 1 fl Interpret:/0/0 rc -22/-22 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764769.738499:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970c00380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+6:5795:x1862022978353152:12345-192.168.202.146@tcp:104: Request processed in 82us (387us total) trans 0 rc -22/-22 00000100:00100000:0.0:1775764769.738507:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24065 00000100:00100000:0.0:1775764769.738521:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24066 00000100:00100000:0.0:1775764769.738527:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970c02d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+5:5795:x1862022978353280:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764769.738538:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971078a00/0x36f9bdc9e421bb75 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x2aac6a50e5f76a1 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764769.738574:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764769.738580:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970c02d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+5:5795:x1862022978353280:12345-192.168.202.146@tcp:104: Request processed in 54us (345us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764769.738586:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24066 00010000:00010000:0.0:1775764769.742166:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1775764769.742175:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49400/0x36f9bdc9e421c006 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f84af expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.742185:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9966d49400 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:0.0:1775764769.742189:0:5571:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49400/0x36f9bdc9e421c006 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d4:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f84af expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.742198:0:5571:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49400/0x36f9bdc9e421c006 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d4:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f84af expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.742206:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49400/0x36f9bdc9e421c006 lrc: 3/0,0 mode: PR/PR res: [0x200000401:0x2d4:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f84af expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.742225:0:5571:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8dc00/0x36f9bdc9e421b7b6 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f6b41 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764769.742262:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49400/0x36f9bdc9e421c006 lrc: 2/0,0 mode: --/PR res: [0x200000401:0x2d4:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f84af expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.742269:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764769.742299:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49400/0x36f9bdc9e421c006 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2d4:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f84af expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.742307:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49400/0x36f9bdc9e421c006 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2d4:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f84af expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.742322:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764769.742324:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764769.742326:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764769.742335:0:2380:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1775764769.742348:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966889600:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764769.758876:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978353408 00000100:00100000:0.0:1775764769.758883:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978353408 00000100:00100000:0.0:1775764769.763559:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00010000:00010000:0.0:1775764769.787288:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1775764769.787295:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49400/0x36f9bdc9e421c022 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f8503 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.787304:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9966d49400 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:0.0:1775764769.787308:0:19801:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49400/0x36f9bdc9e421c022 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d5:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f8503 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.787316:0:19801:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49400/0x36f9bdc9e421c022 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d5:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f8503 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.787324:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49400/0x36f9bdc9e421c022 lrc: 3/0,0 mode: PR/PR res: [0x200000401:0x2d5:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f8503 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.787342:0:19801:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971078600/0x36f9bdc9e421bba6 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f7726 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764769.787369:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49400/0x36f9bdc9e421c022 lrc: 2/0,0 mode: --/PR res: [0x200000401:0x2d5:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8503 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.787375:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764769.787412:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49400/0x36f9bdc9e421c022 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2d5:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8503 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.787419:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49400/0x36f9bdc9e421c022 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2d5:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8503 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.787432:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764769.787434:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764769.787435:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764769.787839:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966890752:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764769.803017:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966890752:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1775764769.810694:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1775764769.810702:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebe00/0x36f9bdc9e421c030 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f852d expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.810709:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99706ebe00 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:0.0:1775764769.810714:0:5571:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebe00/0x36f9bdc9e421c030 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e0:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f852d expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.810721:0:5571:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebe00/0x36f9bdc9e421c030 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e0:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f852d expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.810729:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebe00/0x36f9bdc9e421c030 lrc: 3/0,0 mode: PR/PR res: [0x200000401:0x2e0:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f852d expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.810761:0:5571:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45c00/0x36f9bdc9e42156d5 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e2c2e expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764769.810787:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebe00/0x36f9bdc9e421c030 lrc: 2/0,0 mode: --/PR res: [0x200000401:0x2e0:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f852d expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.810794:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764769.810817:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebe00/0x36f9bdc9e421c030 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2e0:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f852d expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.810824:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebe00/0x36f9bdc9e421c030 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2e0:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f852d expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.810840:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764769.810842:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764769.810844:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764769.810854:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966891520:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764769.816345:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764769.816383:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764769.816400:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764769.816423:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978354048 00000100:00100000:0.0:1775764769.816431:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978354048 00000100:00100000:0.0:1775764769.816469:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978354176 00000100:00100000:0.0:1775764769.816472:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978354176 00000100:00100000:0.0:1775764769.816479:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978354304 00000100:00100000:0.0:1775764769.816481:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978354304 00000100:00100000:0.0:1775764769.816492:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24072 00000100:00100000:0.0:1775764769.816501:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e7100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+7:16259:x1862022978354048:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764769.816512:0:5561:0:(ldlm_lockd.c:2448:ldlm_callback_handler()) callback on lock 0x36f9bdc9e421c030 - lock disappeared 00010000:00010000:0.0:1775764769.816562:0:5561:0:(ldlm_lockd.c:2345:ldlm_callback_errmsg()) @@@ Operate with invalid parameter, NID=12345-192.168.202.146@tcp lock=0x36f9bdc9e421c030: rc = 0 req@ffff8f994f2e7100 x1862022978354048/t0(0) o104->3f83546c-01a9-4156-8a97-44c446245d3e@192.168.202.146@tcp:250/0 lens 328/224 e 0 to 0 dl 1775764780 ref 1 fl Interpret:/0/0 rc -22/-22 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764769.816573:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e7100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+7:16259:x1862022978354048:12345-192.168.202.146@tcp:104: Request processed in 74us (240us total) trans 0 rc -22/-22 00000100:00100000:0.0:1775764769.816580:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24072 00000100:00100000:0.0:1775764769.816596:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24073 00000100:00100000:0.0:1775764769.816601:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e4380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+6:16259:x1862022978354176:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764769.816614:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45c00/0x36f9bdc9e42156d5 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x2aac6a50e5e2c2e expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764769.816677:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764769.816683:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e4380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+6:16259:x1862022978354176:12345-192.168.202.146@tcp:104: Request processed in 82us (299us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764769.816690:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24073 00000100:00100000:0.0:1775764769.816702:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24074 00000100:00100000:0.0:1775764769.816708:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e4a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+5:11866:x1862022978354304:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764769.816716:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970924400/0x36f9bdc9e421c029 lrc: 2/0,0 mode: CW/CW res: [0x200000401:0x2d5:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x2aac6a50e5f8511 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764769.816790:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:0.0:1775764769.816796:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e4a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+5:11866:x1862022978354304:12345-192.168.202.146@tcp:104: Request processed in 89us (395us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764769.816802:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24074 00000100:00100000:0.0:1775764769.816881:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966891520:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764769.835983:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764769.838782:0:11146:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978354560 00000100:00100000:0.0:1775764769.838789:0:11146:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978354560 00000100:00100000:0.0:1775764769.838826:0:11146:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24076 00000100:00100000:0.0:1775764769.838834:0:11146:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970c00380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:3f83546c-01a9-4156-8a97-44c446245d3e+5:11866:x1862022978354560:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764769.838848:0:11146:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970abe600/0x36f9bdc9e421bbc9 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d7:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x2aac6a50e5f77a4 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764769.838906:0:11146:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764769.838912:0:11146:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970c00380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:3f83546c-01a9-4156-8a97-44c446245d3e+5:11866:x1862022978354560:12345-192.168.202.146@tcp:104: Request processed in 79us (404us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764769.838919:0:11146:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24076 00010000:00010000:0.0:1775764769.859659:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1775764769.859667:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49400/0x36f9bdc9e421c061 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f85c7 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.859676:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9966d49400 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:0.0:1775764769.859680:0:5571:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49400/0x36f9bdc9e421c061 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e2:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f85c7 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.859694:0:5571:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49400/0x36f9bdc9e421c061 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e2:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f85c7 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.859702:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49400/0x36f9bdc9e421c061 lrc: 3/0,0 mode: PR/PR res: [0x200000401:0x2e2:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f85c7 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.859718:0:5571:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970abf000/0x36f9bdc9e421592f lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e335f expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764769.859769:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49400/0x36f9bdc9e421c061 lrc: 2/0,0 mode: --/PR res: [0x200000401:0x2e2:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f85c7 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.859775:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764769.859793:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49400/0x36f9bdc9e421c061 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2e2:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f85c7 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.859799:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49400/0x36f9bdc9e421c061 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2e2:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f85c7 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.859814:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764769.859815:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764769.859817:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764769.859828:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966893440:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764769.865378:0:5561:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1775764769.865832:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978354688 00000100:00100000:0.0:1775764769.865840:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978354688 00000100:00100000:0.0:1775764769.865890:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24077 00000100:00100000:0.0:1775764769.865899:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970e0a050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+5:5794:x1862022978354688:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764769.865916:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8cc00/0x36f9bdc9e421c053 lrc: 2/0,0 mode: CW/CW res: [0x200000401:0x2d7:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x2aac6a50e5f858f expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764769.865985:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764769.865991:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970e0a050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+5:5794:x1862022978354688:12345-192.168.202.146@tcp:104: Request processed in 93us (2834us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764769.865997:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24077 00000100:00100000:0.0:1775764769.874354:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966893440:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1775764769.883051:0:5572:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1775764769.883057:0:5572:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8da00/0x36f9bdc9e421c076 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f85ff expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.883065:0:5572:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f994ed8da00 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:0.0:1775764769.883070:0:5572:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8da00/0x36f9bdc9e421c076 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d9:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f85ff expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.883079:0:5572:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8da00/0x36f9bdc9e421c076 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d9:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f85ff expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.883087:0:5572:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8da00/0x36f9bdc9e421c076 lrc: 3/0,0 mode: PR/PR res: [0x200000401:0x2d9:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f85ff expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.884405:0:5572:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093d400/0x36f9bdc9e421bbf3 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f781b expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764769.884434:0:5572:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8da00/0x36f9bdc9e421c076 lrc: 2/0,0 mode: --/PR res: [0x200000401:0x2d9:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f85ff expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.884441:0:5572:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764769.884459:0:5572:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8da00/0x36f9bdc9e421c076 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2d9:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f85ff expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.884466:0:5572:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8da00/0x36f9bdc9e421c076 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2d9:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f85ff expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.884479:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764769.884481:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764769.884482:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764769.884492:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966894080:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764769.888045:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764769.918184:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764769.918225:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00010000:00010000:1.0:1775764769.918542:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b8400/0x36f9bdc9e421573e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x2aac6a50e5e2d54 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764769.918583:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764769.918588:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9966932850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+5:5794:x1862022978355584:12345-192.168.202.146@tcp:104: Request processed in 60us (366us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764769.918594:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24084 00000100:00100000:0.0:1775764769.923595:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970c00700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966895232:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764769.923608:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970c03100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966895360:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764769.933569:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764769.933827:0:11146:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978355712 00000100:00100000:1.0:1775764769.933837:0:11146:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978355712 00000100:00100000:1.0:1775764769.933888:0:11146:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24085 00000100:00100000:1.0:1775764769.933898:0:11146:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e4a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:3f83546c-01a9-4156-8a97-44c446245d3e+5:5795:x1862022978355712:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764769.933917:0:11146:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d9400/0x36f9bdc9e421c099 lrc: 2/0,0 mode: CW/CW res: [0x200000401:0x2e4:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x2aac6a50e5f8661 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764769.934020:0:11146:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764769.934027:0:11146:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e4a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:3f83546c-01a9-4156-8a97-44c446245d3e+5:5795:x1862022978355712:12345-192.168.202.146@tcp:104: Request processed in 131us (469us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764769.934035:0:11146:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24085 00000100:00100000:1.0:1775764769.938796:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703dfb80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966895744:192.168.202.146@tcp:101:dbench.0 00010000:00010000:1.0:1775764769.938819:0:19723:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8da00/0x36f9bdc9e421c0a7 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2e4:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8692 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764769.938831:0:19723:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8da00/0x36f9bdc9e421c0a7 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2e4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8692 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764769.938842:0:19723:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8da00/0x36f9bdc9e421c0a7 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2e4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8692 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764769.938857:0:19723:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8da00/0x36f9bdc9e421c0a7 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2e4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8692 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764769.938869:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8da00/0x36f9bdc9e421c0a7 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2e4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8692 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764769.938877:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8da00/0x36f9bdc9e421c0a7 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2e4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8692 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764769.938883:0:19723:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8da00/0x36f9bdc9e421c0a7 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2e4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8692 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764769.938893:0:19723:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f99703dfb80 x1862022966895744/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/928 e 0 to 0 dl 1775764785 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'dbench.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764769.938904:0:19723:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8da00/0x36f9bdc9e421c0a7 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2e4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8692 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1775764769.938918:0:19723:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8da00/0x36f9bdc9e421c0a7 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2e4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8692 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764769.938932:0:19723:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/740 (0 0) 00000080:00200000:1.0:1775764769.938937:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2e4:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1775764769.939045:0:19723:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x2e4:0x0]: layout version change: 4294967294 -> 3 00000080:00010000:1.0:1775764769.939056:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2e4:0x0](ffff8f9952d900c8) for lock 0x36f9bdc9e421c0a7 00000080:00010000:1.0:1775764769.939062:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2e4:0x0](ffff8f9952d900c8) 00000080:00010000:1.0:1775764769.939065:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c0a7 from it ffffafbac1c7bcd8 00010000:00010000:1.0:1775764769.939069:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8da00/0x36f9bdc9e421c0a7 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2e4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8692 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764769.939077:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8da00/0x36f9bdc9e421c0a7 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8692 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764769.939144:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2e4:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764769.939154:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2e4:0x0] 00000008:00010000:1.0:1775764769.939181:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764769.939185:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99705eeb30, osc lock ffff8f995875c5f0, flags 0x400001002 00010000:00010000:1.0:1775764769.939202:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000e953e3b/0x36f9bdc9e41cb0ce lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x68:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef5cd expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 0 00010000:00010000:1.0:1775764769.939216:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000e953e3b/0x36f9bdc9e41cb0ce lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x68:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef5cd expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 0 00000020:00200000:1.0:1775764769.939228:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764769.939232:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000e953e3b/0x36f9bdc9e41cb0ce lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x68:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ef5cd expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 0 00010000:00010000:1.0:1775764769.939240:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000e953e3b/0x36f9bdc9e41cb0ce lrc: 4/0,0 mode: PW/PW res: [0x240000403:0x68:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef5cd expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 0 00000020:00200000:1.0:1775764769.939251:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764769.939259:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2e4:0x0] updating i_size 260096 i_blocks 1 00000020:00010000:1.0:1775764769.939263:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705f9240 (ffff8f99705b8098/0) at cl_lock_release():245 00000020:00010000:1.0:1775764769.939266:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705f9240 (ffff8f99705b8098/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764769.939269:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705eeb30 (ffff8f99705b87e8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764769.939274:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705f9240 (ffff8f99705b8098/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764769.939276:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705eeb30 (ffff8f99705b8098/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764769.939292:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2e4:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764769.939355:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.939361:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.939367:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:1.0:1775764769.939373:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.939377:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:1.0:1775764769.939380:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764769.939383:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:1.0:1775764769.939385:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764769.939389:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=*, flags=1 00000080:00200000:1.0:1775764769.939427:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.939430:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.939433:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=19 00000080:00200000:1.0:1775764769.939436:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.939438:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=19 00000080:00200000:1.0:1775764769.939440:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764769.939444:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=259 00000080:00200000:1.0:1775764769.939448:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 44 00000080:00200008:1.0:1775764769.939455:0:19723:0:(file.c:1057:ll_file_open()) START file ~dmtmp:[0x200000401:0x26c:0x0](ffff8f9952db6c08), flags 304000 00000080:00200008:1.0:1775764769.939471:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file ~dmtmp:[0x200000401:0x26c:0x0](ffff8f9952db6c08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764769.939476:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08),name=~dmtmp 00000002:00010000:1.0:1775764769.939493:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x26c:0x0]) in obj [0x200000401:0x26a:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764769.939503:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764769.939511:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764769.939518:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26a:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764769.939521:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x26c:0x0](ffff8f9952db6c08) 00000080:00010000:1.0:1775764769.939524:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x26c:0x0](ffff8f9952db6c08) for lock 0x36f9bdc9e41cb2b1 00000080:00010000:1.0:1775764769.939527:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e41cb2b1 from it ffffafbac1c7bcd8 00010000:00010000:1.0:1775764769.939530:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764769.939537:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764769.939554:0:19723:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08) pos/size0/7168 32bit_api 0 00000080:00200000:1.0:1775764769.939579:0:19723:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08) pos/size18446744073709551614/7168 32bit_api 0 00000080:00200008:1.0:1775764769.939591:0:19723:0:(file.c:447:ll_file_release()) START file ~dmtmp:[0x200000401:0x26c:0x0](ffff8f9952db6c08), flags 304000 00010000:00010000:1.0:1775764769.939602:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971078400/0x36f9bdc9e41ca50c lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ed7b9 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764769.939622:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.939637:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.939641:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=19 00000080:00200000:1.0:1775764769.939644:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.939646:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=19 00000080:00200000:1.0:1775764769.939648:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764769.939651:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=259 00000080:00200000:1.0:1775764769.939653:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 44 00000080:00200008:1.0:1775764769.939657:0:19723:0:(file.c:1057:ll_file_open()) START file ~dmtmp:[0x200000401:0x26c:0x0](ffff8f9952db6c08), flags 304000 00000080:00200008:1.0:1775764769.939665:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file ~dmtmp:[0x200000401:0x26c:0x0](ffff8f9952db6c08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764769.939670:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08),name=~dmtmp 00000002:00010000:1.0:1775764769.939677:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x26c:0x0]) in obj [0x200000401:0x26a:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764769.939683:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764769.939690:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764769.939696:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26a:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764769.939699:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x26c:0x0](ffff8f9952db6c08) 00000080:00010000:1.0:1775764769.939702:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x26c:0x0](ffff8f9952db6c08) for lock 0x36f9bdc9e41cb2b1 00000080:00010000:1.0:1775764769.939705:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e41cb2b1 from it ffffafbac1c7bcd8 00010000:00010000:1.0:1775764769.939708:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764769.939714:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764769.939724:0:19723:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08) pos/size0/7168 32bit_api 0 00000080:00200000:1.0:1775764769.939734:0:19723:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08) pos/size18446744073709551614/7168 32bit_api 0 00000080:00200008:1.0:1775764769.939764:0:19723:0:(file.c:447:ll_file_release()) START file ~dmtmp:[0x200000401:0x26c:0x0](ffff8f9952db6c08), flags 304000 00010000:00010000:1.0:1775764769.939771:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971078400/0x36f9bdc9e41ca50c lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ed7b9 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200008:1.0:1775764769.939795:0:19723:0:(file.c:2454:do_file_read_iter()) START file COURSES.FSL:[0x200000401:0x2e4:0x0], ppos: 0, count: 4096 00000080:00200008:1.0:1775764769.939814:0:19723:0:(file.c:2454:do_file_read_iter()) START file COURSES.FSL:[0x200000401:0x2e4:0x0], ppos: 4096, count: 4096 00000080:00200000:1.0:1775764769.939828:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.939831:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.939835:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:1.0:1775764769.939838:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.939840:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:1.0:1775764769.939842:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764769.939845:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:1.0:1775764769.939848:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764769.939851:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:1.0:1775764769.939853:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764769.939865:0:19723:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=COURSES.DB, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), flags=1 00000080:00200000:1.0:1775764769.939869:0:19723:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=COURSES.DB, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), intent=getattr 00000002:00010000:1.0:1775764769.939879:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: COURSES.DB,[0x0:0x0:0x0]) in obj [0x200000401:0x2bb:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764769.939918:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c0ae lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764769.939925:0:19723:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c0ae lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764769.939933:0:19723:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c0ae lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764769.939955:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970c02d80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966896000:192.168.202.146@tcp:101:dbench.0 00000100:00100000:1.0:1775764769.940032:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f997130dd80 going to sleep for 11 seconds 00010000:00010000:0.0:1775764769.947677:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1775764769.947689:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d9600/0x36f9bdc9e421c0a0 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2cf:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f867d expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.947700:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f994f3d9600 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:0.0:1775764769.947706:0:19801:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d9600/0x36f9bdc9e421c0a0 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2cf:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f867d expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.947720:0:19801:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d9600/0x36f9bdc9e421c0a0 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2cf:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f867d expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.947729:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d9600/0x36f9bdc9e421c0a0 lrc: 3/0,0 mode: PR/PR res: [0x200000401:0x2cf:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f867d expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.947796:0:19801:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fc000/0x36f9bdc9e421bc0f lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2cf:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f787d expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764769.947842:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d9600/0x36f9bdc9e421c0a0 lrc: 2/0,0 mode: --/PR res: [0x200000401:0x2cf:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f867d expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.947848:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764769.947880:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d9600/0x36f9bdc9e421c0a0 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2cf:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f867d expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.947887:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d9600/0x36f9bdc9e421c0a0 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2cf:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f867d expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764769.947907:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764769.947909:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764769.947911:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764769.947922:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1775764769.947959:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966896384:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764769.952458:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764769.952488:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978355840 00000100:00100000:0.0:1775764769.952494:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978355840 00000100:00100000:0.0:1775764769.952539:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24086 00000100:00100000:0.0:1775764769.952549:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e6680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+5:5794:x1862022978355840:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764769.952558:0:5562:0:(ldlm_lockd.c:2448:ldlm_callback_handler()) callback on lock 0x36f9bdc9e421c0a0 - lock disappeared 00010000:00010000:0.0:1775764769.952606:0:5562:0:(ldlm_lockd.c:2345:ldlm_callback_errmsg()) @@@ Operate with invalid parameter, NID=12345-192.168.202.146@tcp lock=0x36f9bdc9e421c0a0: rc = 0 req@ffff8f994f2e6680 x1862022978355840/t0(0) o104->3f83546c-01a9-4156-8a97-44c446245d3e@192.168.202.146@tcp:250/0 lens 328/224 e 0 to 0 dl 1775764780 ref 1 fl Interpret:/0/0 rc -22/-22 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764769.952636:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e6680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+5:5794:x1862022978355840:12345-192.168.202.146@tcp:104: Request processed in 81us (172us total) trans 0 rc -22/-22 00000100:00100000:0.0:1775764769.952644:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24086 00000100:00100000:0.0:1775764769.952679:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764769.952697:0:11146:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978355968 00000100:00100000:0.0:1775764769.952701:0:11146:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978355968 00000100:00100000:0.0:1775764769.952718:0:11146:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24087 00000100:00100000:0.0:1775764769.952724:0:11146:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e6a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:3f83546c-01a9-4156-8a97-44c446245d3e+5:5794:x1862022978355968:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764769.952733:0:11146:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fc000/0x36f9bdc9e421bc0f lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2cf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x2aac6a50e5f787d expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764769.952789:0:11146:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764769.952794:0:11146:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e6a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:3f83546c-01a9-4156-8a97-44c446245d3e+5:5794:x1862022978355968:12345-192.168.202.146@tcp:104: Request processed in 71us (115us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764769.952800:0:11146:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24087 00000002:00010000:1.0:1775764769.953101:0:19723:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c0ae lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2bd:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f86ae expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764769.953118:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970972a00/0x36f9bdc9e421bd5f lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2bd:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f7ca5 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764769.953130:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970972a00/0x36f9bdc9e421bd5f lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2bd:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f7ca5 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764769.953143:0:19723:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c0ae lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2bd:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f86ae expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764769.953149:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c0ae lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2bd:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x2aac6a50e5f86ae expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764769.953156:0:19723:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c0ae lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2bd:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x2aac6a50e5f86ae expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764769.953178:0:19723:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00000080:00200000:1.0:1775764769.953184:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2bb:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1775764769.953262:0:19723:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x2bd:0x0](ffff8f9952d14b08): rc = 0 00000080:00200000:1.0:1775764769.953282:0:19723:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x2bd:0x0]: layout version change: 3 -> 3 00000080:00010000:1.0:1775764769.953292:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2bd:0x0](ffff8f9952d14b08) for lock 0x36f9bdc9e421bd5f 00000080:00010000:1.0:1775764769.953305:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2bd:0x0](ffff8f9952d14b08) 00000080:00010000:1.0:1775764769.953309:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421bd5f from it ffffafbac1c7bb20 00010000:00010000:1.0:1775764769.953312:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970972a00/0x36f9bdc9e421bd5f lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2bd:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f7ca5 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764769.953322:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970972a00/0x36f9bdc9e421bd5f lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2bd:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f7ca5 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200008:1.0:1775764769.953382:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file COURSES.DB:[0x200000401:0x2bd:0x0](ffff8f9952d14b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764769.953388:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2bd:0x0](ffff8f9952d14b08),name=COURSES.DB 00000002:00010000:1.0:1775764769.953417:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2bd:0x0]) in obj [0x200000401:0x2bb:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764769.953426:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970972a00/0x36f9bdc9e421bd5f lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2bd:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f7ca5 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764769.953435:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970972a00/0x36f9bdc9e421bd5f lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2bd:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f7ca5 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764769.953441:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2bb:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764769.953445:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2bd:0x0](ffff8f9952d14b08) 00000080:00010000:1.0:1775764769.953448:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2bd:0x0](ffff8f9952d14b08) for lock 0x36f9bdc9e421bd5f 00000080:00010000:1.0:1775764769.953452:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421bd5f from it ffffafbac1c7bcb8 00010000:00010000:1.0:1775764769.953454:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970972a00/0x36f9bdc9e421bd5f lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2bd:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f7ca5 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764769.953460:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970972a00/0x36f9bdc9e421bd5f lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2bd:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f7ca5 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764769.953484:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2bd:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764769.953493:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2bd:0x0] 00000008:00010000:1.0:1775764769.953517:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764769.953520:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99705eeb30, osc lock ffff8f995875c5f0, flags 0x400001002 00010000:00010000:1.0:1775764769.953535:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003cb7abbf/0x36f9bdc9e41cab72 lrc: 3/0,1 mode: PW/PW res: [0x240000403:0x55:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee8de expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 79 lru_type: 1 00010000:00010000:1.0:1775764769.953546:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003cb7abbf/0x36f9bdc9e41cab72 lrc: 3/0,1 mode: PW/PW res: [0x240000403:0x55:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee8de expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 79 lru_type: 1 00000020:00200000:1.0:1775764769.953557:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764769.953560:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003cb7abbf/0x36f9bdc9e41cab72 lrc: 6/0,1 mode: PW/PW res: [0x240000403:0x55:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ee8de expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 79 lru_type: 1 00010000:00010000:1.0:1775764769.953573:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003cb7abbf/0x36f9bdc9e41cab72 lrc: 5/0,0 mode: PW/PW res: [0x240000403:0x55:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee8de expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 80 lru_type: 1 00000020:00200000:1.0:1775764769.953582:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764769.953590:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2bd:0x0] updating i_size 260096 i_blocks 515 00000020:00010000:1.0:1775764769.953593:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705f9240 (ffff8f99705b8098/0) at cl_lock_release():245 00000020:00010000:1.0:1775764769.953596:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705f9240 (ffff8f99705b8098/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764769.953599:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705eeb30 (ffff8f99705b87e8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764769.953602:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705f9240 (ffff8f99705b8098/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764769.953604:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705eeb30 (ffff8f99705b8098/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764769.953638:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2bd:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1775764769.953693:0:19723:0:(file.c:2612:do_file_write_iter()) START file COURSES.DB:[0x200000401:0x2bd:0x0], ppos: 0, count: 4096 00000080:00200000:1.0:1775764769.953709:0:19723:0:(file.c:2058:ll_file_io_generic()) COURSES.DB: write ppos: 0, bytes: 4096 00000080:00200000:1.0:1775764769.953714:0:19723:0:(file.c:1815:ll_io_set_mirror()) COURSES.DB: desiginated mirror: 0 00000020:00200000:1.0:1775764769.953720:0:19723:0:(cl_io.c:214:cl_io_rw_init()) header@00000000614e5365[0x110, 65, [0x200000401:0x2bd:0x0] hash] 00000020:00200000:1.0:1775764769.953722:0:19723:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 4096) 0 0 00000080:00200000:1.0:1775764769.953725:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2bd:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764769.953734:0:19723:0:(file.c:2136:ll_file_io_generic()) Range lock [0, 0] 00020000:00200000:1.0:1775764769.953763:0:19723:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 4096) 4096, 4096 00020000:00200000:1.0:1775764769.953768:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764769.953787:0:19723:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:1.0:1775764769.953794:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764769.953797:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99705eeb30, osc lock ffff8f995875c5f0, flags 0x0 00010000:00010000:1.0:1775764769.953806:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003cb7abbf/0x36f9bdc9e41cab72 lrc: 3/0,1 mode: PW/PW res: [0x240000403:0x55:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee8de expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 80 lru_type: 1 00010000:00010000:1.0:1775764769.953815:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003cb7abbf/0x36f9bdc9e41cab72 lrc: 3/0,1 mode: PW/PW res: [0x240000403:0x55:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee8de expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 80 lru_type: 1 00010000:00010000:1.0:1775764769.953825:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003cb7abbf/0x36f9bdc9e41cab72 lrc: 7/0,2 mode: PW/PW res: [0x240000403:0x55:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ee8de expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 80 lru_type: 1 00010000:00010000:1.0:1775764769.953834:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003cb7abbf/0x36f9bdc9e41cab72 lrc: 7/0,2 mode: PW/PW res: [0x240000403:0x55:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ee8de expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 80 lru_type: 1 00010000:00010000:1.0:1775764769.953841:0:19723:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003cb7abbf/0x36f9bdc9e41cab72 lrc: 6/0,1 mode: PW/PW res: [0x240000403:0x55:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ee8de expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 80 lru_type: 1 00000080:00200000:1.0:1775764769.953852:0:19723:0:(vvp_io.c:1286:vvp_io_write_start()) COURSES.DB: write [0, 4096) append 0 append lockpos 0 00000080:00200000:1.0:1775764769.953865:0:19723:0:(rw26.c:663:ll_write_begin()) Writing 0 of 0 to 4096 bytes 00000080:00200000:1.0:1775764769.953881:0:19723:0:(rw26.c:858:ll_write_end()) pos 0, len 4096, copied 4096 00000080:00200000:1.0:1775764769.953887:0:19723:0:(rw26.c:916:ll_write_end()) page@000000003aa73994[4 00000000a0548882 2 1 000000006a6d0f1a] 00000080:00200000:1.0:1775764769.953891:0:19723:0:(rw26.c:916:ll_write_end()) vmpage @00000000ad4b6846 17ffffc000107d 2:0 ffff8f99709f82a0 0 lru 00000080:00200000:1.0:1775764769.953899:0:19723:0:(rw26.c:916:ll_write_end()) osc-page@000000000d465ab8 0: 1< 2 + - > 2< 0 0 4096 0x0 0x40420 | 00000000f240ca6f 000000006b8eb50b > 3< 1 0 > 4< 0 0 8 412352512 - | - - + - > 5< - - + - | 0 - | 2 - -> 00000080:00200000:1.0:1775764769.953901:0:19723:0:(rw26.c:916:ll_write_end()) end page@000000003aa73994 00000080:00200000:1.0:1775764769.953903:0:19723:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764769.953905:0:19723:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000080:00200000:1.0:1775764769.953917:0:19723:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8f9952d14cb8, count 1, dirtied 0 00000080:00200000:1.0:1775764769.953921:0:19723:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764769.953925:0:19723:0:(vvp_io.c:1353:vvp_io_write_start()) COURSES.DB: write bytes 0, result: 4096 00000020:00010000:1.0:1775764769.953930:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705bae48 (ffff8f99705b8098/0) at cl_lock_release():245 00000020:00010000:1.0:1775764769.953933:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705bae48 (ffff8f99705b8098/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764769.953935:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705eeb30 (ffff8f99705b87e8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764769.953939:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003cb7abbf/0x36f9bdc9e41cab72 lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x55:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ee8de expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 80 lru_type: 1 00010000:00010000:1.0:1775764769.953951:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003cb7abbf/0x36f9bdc9e41cab72 lrc: 4/0,0 mode: PW/PW res: [0x240000403:0x55:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee8de expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 81 lru_type: 1 00000020:00010000:1.0:1775764769.953960:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705bae48 (ffff8f99705b8098/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764769.953962:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705eeb30 (ffff8f99705b8098/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764769.953967:0:19723:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764769.953973:0:19723:0:(file.c:2173:ll_file_io_generic()) Range unlock [0, 0] 00000080:00200000:1.0:1775764769.953986:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2bd:0x0] ignore/verify layout 0/0, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764769.953989:0:19723:0:(file.c:2206:ll_file_io_generic()) COURSES.DB: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764769.953995:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2bd:0x0] updating i_size 260096 i_blocks 515 00000080:00200000:1.0:1775764769.953999:0:19723:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764769.954022:0:19723:0:(file.c:2612:do_file_write_iter()) START file COURSES.PX:[0x200000401:0x2c0:0x0], ppos: 0, count: 4096 00000080:00200000:1.0:1775764769.954034:0:19723:0:(file.c:2058:ll_file_io_generic()) COURSES.PX: write ppos: 0, bytes: 4096 00000080:00200000:1.0:1775764769.954037:0:19723:0:(file.c:1815:ll_io_set_mirror()) COURSES.PX: desiginated mirror: 0 00000020:00200000:1.0:1775764769.954041:0:19723:0:(cl_io.c:214:cl_io_rw_init()) header@000000006b3b75f5[0x110, 65, [0x200000401:0x2c0:0x0] hash] 00000020:00200000:1.0:1775764769.954042:0:19723:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 4096) 0 0 00000080:00200000:1.0:1775764769.954044:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2c0:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764769.954051:0:19723:0:(file.c:2136:ll_file_io_generic()) Range lock [0, 0] 00020000:00200000:1.0:1775764769.954054:0:19723:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 4096) 4096, 4096 00020000:00200000:1.0:1775764769.954056:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764769.954070:0:19723:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:1.0:1775764769.954075:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764769.954077:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99705eeb30, osc lock ffff8f995875c5f0, flags 0x0 00010000:00010000:1.0:1775764769.954088:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000ceb22e30/0x36f9bdc9e41cabd4 lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x56:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee9b0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 0 00010000:00010000:1.0:1775764769.954097:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000ceb22e30/0x36f9bdc9e41cabd4 lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x56:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee9b0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 0 00010000:00010000:1.0:1775764769.954111:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000ceb22e30/0x36f9bdc9e41cabd4 lrc: 7/0,2 mode: PW/PW res: [0x280000403:0x56:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ee9b0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 0 00010000:00010000:1.0:1775764769.954118:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000ceb22e30/0x36f9bdc9e41cabd4 lrc: 7/0,2 mode: PW/PW res: [0x280000403:0x56:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ee9b0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 0 00010000:00010000:1.0:1775764769.954125:0:19723:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000ceb22e30/0x36f9bdc9e41cabd4 lrc: 6/0,1 mode: PW/PW res: [0x280000403:0x56:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ee9b0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 0 00000080:00200000:1.0:1775764769.954132:0:19723:0:(vvp_io.c:1286:vvp_io_write_start()) COURSES.PX: write [0, 4096) append 0 append lockpos 0 00000080:00200000:1.0:1775764769.954138:0:19723:0:(rw26.c:663:ll_write_begin()) Writing 0 of 0 to 4096 bytes 00000080:00200000:1.0:1775764769.954144:0:19723:0:(rw26.c:858:ll_write_end()) pos 0, len 4096, copied 4096 00000080:00200000:1.0:1775764769.954148:0:19723:0:(rw26.c:916:ll_write_end()) page@00000000558ccef4[4 000000008a4ce302 2 1 000000006a6d0f1a] 00000080:00200000:1.0:1775764769.954150:0:19723:0:(rw26.c:916:ll_write_end()) vmpage @0000000013d6400c 17ffffc000107d 2:0 ffff8f994f20e0e0 0 lru 00000080:00200000:1.0:1775764769.954157:0:19723:0:(rw26.c:916:ll_write_end()) osc-page@000000001de21bf4 0: 1< 2 + - > 2< 0 0 4096 0x0 0x40420 | 000000004b957d6e 0000000056ce2969 > 3< 1 0 > 4< 0 0 8 457834496 - | - - + - > 5< - - + - | 0 - | 1 - -> 00000080:00200000:1.0:1775764769.954159:0:19723:0:(rw26.c:916:ll_write_end()) end page@00000000558ccef4 00000080:00200000:1.0:1775764769.954160:0:19723:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764769.954162:0:19723:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000080:00200000:1.0:1775764769.954167:0:19723:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8f9952d8abb8, count 1, dirtied 0 00000080:00200000:1.0:1775764769.954169:0:19723:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764769.954172:0:19723:0:(vvp_io.c:1353:vvp_io_write_start()) COURSES.PX: write bytes 0, result: 4096 00000020:00010000:1.0:1775764769.954175:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705bae48 (ffff8f99705b8098/0) at cl_lock_release():245 00000020:00010000:1.0:1775764769.954178:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705bae48 (ffff8f99705b8098/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764769.954181:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705eeb30 (ffff8f99705b87e8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764769.954185:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000ceb22e30/0x36f9bdc9e41cabd4 lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x56:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ee9b0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 0 00010000:00010000:1.0:1775764769.954200:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000ceb22e30/0x36f9bdc9e41cabd4 lrc: 4/0,0 mode: PW/PW res: [0x280000403:0x56:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee9b0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 0 00000020:00010000:1.0:1775764769.954209:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705bae48 (ffff8f99705b8098/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764769.954212:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705eeb30 (ffff8f99705b8098/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764769.954216:0:19723:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764769.954219:0:19723:0:(file.c:2173:ll_file_io_generic()) Range unlock [0, 0] 00000080:00200000:1.0:1775764769.954230:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2c0:0x0] ignore/verify layout 0/0, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764769.954233:0:19723:0:(file.c:2206:ll_file_io_generic()) COURSES.PX: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764769.954238:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2c0:0x0] updating i_size 260096 i_blocks 515 00000080:00200000:1.0:1775764769.954241:0:19723:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764769.954258:0:19723:0:(file.c:2612:do_file_write_iter()) START file COURSES.X04:[0x200000401:0x2c2:0x0], ppos: 0, count: 4096 00000080:00200000:1.0:1775764769.954269:0:19723:0:(file.c:2058:ll_file_io_generic()) COURSES.X04: write ppos: 0, bytes: 4096 00000080:00200000:1.0:1775764769.954272:0:19723:0:(file.c:1815:ll_io_set_mirror()) COURSES.X04: desiginated mirror: 0 00000020:00200000:1.0:1775764769.954276:0:19723:0:(cl_io.c:214:cl_io_rw_init()) header@000000005b7d5a48[0x110, 65, [0x200000401:0x2c2:0x0] hash] 00000020:00200000:1.0:1775764769.954277:0:19723:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 4096) 0 0 00000080:00200000:1.0:1775764769.954280:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2c2:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764769.954286:0:19723:0:(file.c:2136:ll_file_io_generic()) Range lock [0, 0] 00020000:00200000:1.0:1775764769.954289:0:19723:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 4096) 4096, 4096 00020000:00200000:1.0:1775764769.954291:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764769.954302:0:19723:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:1.0:1775764769.954307:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764769.954309:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99705eeb30, osc lock ffff8f995875c5f0, flags 0x0 00010000:00010000:1.0:1775764769.954317:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000047343c86/0x36f9bdc9e41cac1a lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x57:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eea58 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 0 00010000:00010000:1.0:1775764769.954325:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000047343c86/0x36f9bdc9e41cac1a lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x57:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eea58 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 0 00010000:00010000:1.0:1775764769.954339:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000047343c86/0x36f9bdc9e41cac1a lrc: 7/0,2 mode: PW/PW res: [0x280000403:0x57:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4eea58 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 0 00010000:00010000:1.0:1775764769.954346:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000047343c86/0x36f9bdc9e41cac1a lrc: 7/0,2 mode: PW/PW res: [0x280000403:0x57:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4eea58 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 0 00010000:00010000:1.0:1775764769.954353:0:19723:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000047343c86/0x36f9bdc9e41cac1a lrc: 6/0,1 mode: PW/PW res: [0x280000403:0x57:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4eea58 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 0 00000080:00200000:1.0:1775764769.954359:0:19723:0:(vvp_io.c:1286:vvp_io_write_start()) COURSES.X04: write [0, 4096) append 0 append lockpos 0 00000080:00200000:1.0:1775764769.954365:0:19723:0:(rw26.c:663:ll_write_begin()) Writing 0 of 0 to 4096 bytes 00000080:00200000:1.0:1775764769.954371:0:19723:0:(rw26.c:858:ll_write_end()) pos 0, len 4096, copied 4096 00000080:00200000:1.0:1775764769.954375:0:19723:0:(rw26.c:916:ll_write_end()) page@000000003b9ea4b1[4 000000002cde6b35 2 1 000000006a6d0f1a] 00000080:00200000:1.0:1775764769.954378:0:19723:0:(rw26.c:916:ll_write_end()) vmpage @000000000434f0cf 17ffffc000107d 2:0 ffff8f9970f91460 0 lru 00000080:00200000:1.0:1775764769.954384:0:19723:0:(rw26.c:916:ll_write_end()) osc-page@00000000243ce695 0: 1< 2 + - > 2< 0 0 4096 0x0 0x40420 | 000000004b957d6e 000000003be2520d > 3< 1 0 > 4< 0 0 8 457834496 - | - - + - > 5< - - + - | 0 - | 1 - -> 00000080:00200000:1.0:1775764769.954386:0:19723:0:(rw26.c:916:ll_write_end()) end page@000000003b9ea4b1 00000080:00200000:1.0:1775764769.954387:0:19723:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764769.954388:0:19723:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000080:00200000:1.0:1775764769.954393:0:19723:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8f9952e2e578, count 1, dirtied 0 00000080:00200000:1.0:1775764769.954395:0:19723:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764769.954398:0:19723:0:(vvp_io.c:1353:vvp_io_write_start()) COURSES.X04: write bytes 0, result: 4096 00000020:00010000:1.0:1775764769.954401:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705bae48 (ffff8f99705b8098/0) at cl_lock_release():245 00000020:00010000:1.0:1775764769.954403:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705bae48 (ffff8f99705b8098/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764769.954405:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705eeb30 (ffff8f99705b87e8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764769.954409:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000047343c86/0x36f9bdc9e41cac1a lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x57:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4eea58 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 0 00010000:00010000:1.0:1775764769.954421:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000047343c86/0x36f9bdc9e41cac1a lrc: 4/0,0 mode: PW/PW res: [0x280000403:0x57:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eea58 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 0 00000020:00010000:1.0:1775764769.954429:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705bae48 (ffff8f99705b8098/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764769.954432:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705eeb30 (ffff8f99705b8098/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764769.954435:0:19723:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764769.954438:0:19723:0:(file.c:2173:ll_file_io_generic()) Range unlock [0, 0] 00000080:00200000:1.0:1775764769.954449:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2c2:0x0] ignore/verify layout 0/0, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764769.954451:0:19723:0:(file.c:2206:ll_file_io_generic()) COURSES.X04: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764769.954456:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2c2:0x0] updating i_size 260096 i_blocks 515 00000080:00200000:1.0:1775764769.954459:0:19723:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764769.954476:0:19723:0:(file.c:2612:do_file_write_iter()) START file COURSES.Y04:[0x200000401:0x2c4:0x0], ppos: 0, count: 4096 00000080:00200000:1.0:1775764769.954489:0:19723:0:(file.c:2058:ll_file_io_generic()) COURSES.Y04: write ppos: 0, bytes: 4096 00000080:00200000:1.0:1775764769.954492:0:19723:0:(file.c:1815:ll_io_set_mirror()) COURSES.Y04: desiginated mirror: 0 00000020:00200000:1.0:1775764769.954496:0:19723:0:(cl_io.c:214:cl_io_rw_init()) header@00000000a800855e[0x110, 65, [0x200000401:0x2c4:0x0] hash] 00000020:00200000:1.0:1775764769.954497:0:19723:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 4096) 0 0 00000080:00200000:1.0:1775764769.954500:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2c4:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764769.954506:0:19723:0:(file.c:2136:ll_file_io_generic()) Range lock [0, 0] 00020000:00200000:1.0:1775764769.954508:0:19723:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 4096) 4096, 4096 00020000:00200000:1.0:1775764769.954511:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764769.954523:0:19723:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:1.0:1775764769.954528:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764769.954531:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99705eeb30, osc lock ffff8f995875c5f0, flags 0x0 00010000:00010000:1.0:1775764769.954539:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000009c24ad6b/0x36f9bdc9e41cac6e lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x58:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eeb23 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 71 lru_type: 1 00010000:00010000:1.0:1775764769.954550:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000009c24ad6b/0x36f9bdc9e41cac6e lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x58:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eeb23 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 71 lru_type: 1 00010000:00010000:1.0:1775764769.954565:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000009c24ad6b/0x36f9bdc9e41cac6e lrc: 7/0,2 mode: PW/PW res: [0x280000403:0x58:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4eeb23 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 71 lru_type: 1 00010000:00010000:1.0:1775764769.954573:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000009c24ad6b/0x36f9bdc9e41cac6e lrc: 7/0,2 mode: PW/PW res: [0x280000403:0x58:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4eeb23 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 71 lru_type: 1 00010000:00010000:1.0:1775764769.954581:0:19723:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000009c24ad6b/0x36f9bdc9e41cac6e lrc: 6/0,1 mode: PW/PW res: [0x280000403:0x58:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4eeb23 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 71 lru_type: 1 00000080:00200000:1.0:1775764769.954589:0:19723:0:(vvp_io.c:1286:vvp_io_write_start()) COURSES.Y04: write [0, 4096) append 0 append lockpos 0 00000080:00200000:1.0:1775764769.954593:0:19723:0:(rw26.c:663:ll_write_begin()) Writing 0 of 0 to 4096 bytes 00000080:00200000:1.0:1775764769.954599:0:19723:0:(rw26.c:858:ll_write_end()) pos 0, len 4096, copied 4096 00000080:00200000:1.0:1775764769.954603:0:19723:0:(rw26.c:916:ll_write_end()) page@0000000098713a96[4 000000001b569f15 2 1 000000006a6d0f1a] 00000080:00200000:1.0:1775764769.954606:0:19723:0:(rw26.c:916:ll_write_end()) vmpage @00000000132772c1 17ffffc000107d 2:0 ffff8f9970b99b60 0 lru 00000080:00200000:1.0:1775764769.954612:0:19723:0:(rw26.c:916:ll_write_end()) osc-page@00000000e9d72131 0: 1< 2 + - > 2< 0 0 4096 0x0 0x40420 | 000000004b957d6e 0000000074dcfdb7 > 3< 1 0 > 4< 0 0 8 457834496 - | - - + - > 5< - - + - | 0 - | 1 - -> 00000080:00200000:1.0:1775764769.954615:0:19723:0:(rw26.c:916:ll_write_end()) end page@0000000098713a96 00000080:00200000:1.0:1775764769.954616:0:19723:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764769.954617:0:19723:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000080:00200000:1.0:1775764769.954622:0:19723:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8f9952e2c478, count 1, dirtied 0 00000080:00200000:1.0:1775764769.954624:0:19723:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764769.954643:0:19723:0:(vvp_io.c:1353:vvp_io_write_start()) COURSES.Y04: write bytes 0, result: 4096 00000020:00010000:1.0:1775764769.954648:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705bae48 (ffff8f99705b8098/0) at cl_lock_release():245 00000020:00010000:1.0:1775764769.954650:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705bae48 (ffff8f99705b8098/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764769.954653:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705eeb30 (ffff8f99705b87e8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764769.954657:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000009c24ad6b/0x36f9bdc9e41cac6e lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x58:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4eeb23 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 71 lru_type: 1 00010000:00010000:1.0:1775764769.954671:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000009c24ad6b/0x36f9bdc9e41cac6e lrc: 4/0,0 mode: PW/PW res: [0x280000403:0x58:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eeb23 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 72 lru_type: 1 00000020:00010000:1.0:1775764769.954679:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705bae48 (ffff8f99705b8098/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764769.954682:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705eeb30 (ffff8f99705b8098/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764769.954686:0:19723:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764769.954690:0:19723:0:(file.c:2173:ll_file_io_generic()) Range unlock [0, 0] 00000080:00200000:1.0:1775764769.954700:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2c4:0x0] ignore/verify layout 0/0, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764769.954703:0:19723:0:(file.c:2206:ll_file_io_generic()) COURSES.Y04: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764769.954708:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2c4:0x0] updating i_size 260096 i_blocks 515 00000080:00200000:1.0:1775764769.954712:0:19723:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200000:1.0:1775764769.954771:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.954777:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.954783:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:1.0:1775764769.954790:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.954793:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:1.0:1775764769.954796:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764769.954799:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:1.0:1775764769.954802:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764769.954807:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=*, flags=1 00000080:00200000:1.0:1775764769.954827:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.954831:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.954833:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=19 00000080:00200000:1.0:1775764769.954836:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.954839:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=19 00000080:00200000:1.0:1775764769.954841:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764769.954843:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=259 00000080:00200000:1.0:1775764769.954848:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 44 00000080:00200008:1.0:1775764769.954854:0:19723:0:(file.c:1057:ll_file_open()) START file ~dmtmp:[0x200000401:0x26c:0x0](ffff8f9952db6c08), flags 304000 00000080:00200008:1.0:1775764769.954870:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file ~dmtmp:[0x200000401:0x26c:0x0](ffff8f9952db6c08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764769.954875:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08),name=~dmtmp 00000002:00010000:1.0:1775764769.954889:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x26c:0x0]) in obj [0x200000401:0x26a:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764769.954900:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764769.954910:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764769.954917:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26a:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764769.954922:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x26c:0x0](ffff8f9952db6c08) 00000080:00010000:1.0:1775764769.954925:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x26c:0x0](ffff8f9952db6c08) for lock 0x36f9bdc9e41cb2b1 00000080:00010000:1.0:1775764769.954930:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e41cb2b1 from it ffffafbac1c7bcd8 00010000:00010000:1.0:1775764769.954932:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764769.954939:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764769.954956:0:19723:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08) pos/size0/7168 32bit_api 0 00000080:00200000:1.0:1775764769.954979:0:19723:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08) pos/size18446744073709551614/7168 32bit_api 0 00000080:00200008:1.0:1775764769.954992:0:19723:0:(file.c:447:ll_file_release()) START file ~dmtmp:[0x200000401:0x26c:0x0](ffff8f9952db6c08), flags 304000 00010000:00010000:1.0:1775764769.955002:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971078400/0x36f9bdc9e41ca50c lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ed7b9 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764769.955024:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.955028:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.955031:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=19 00000080:00200000:1.0:1775764769.955034:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.955036:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=19 00000080:00200000:1.0:1775764769.955039:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764769.955041:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=259 00000080:00200000:1.0:1775764769.955044:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 44 00000080:00200008:1.0:1775764769.955048:0:19723:0:(file.c:1057:ll_file_open()) START file ~dmtmp:[0x200000401:0x26c:0x0](ffff8f9952db6c08), flags 304000 00000080:00200008:1.0:1775764769.955055:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file ~dmtmp:[0x200000401:0x26c:0x0](ffff8f9952db6c08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764769.955059:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08),name=~dmtmp 00000002:00010000:1.0:1775764769.955066:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x26c:0x0]) in obj [0x200000401:0x26a:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764769.955072:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764769.955080:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764769.955087:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26a:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764769.955091:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x26c:0x0](ffff8f9952db6c08) 00000080:00010000:1.0:1775764769.955093:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x26c:0x0](ffff8f9952db6c08) for lock 0x36f9bdc9e41cb2b1 00000080:00010000:1.0:1775764769.955097:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e41cb2b1 from it ffffafbac1c7bcd8 00010000:00010000:1.0:1775764769.955099:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764769.955107:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764769.955118:0:19723:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08) pos/size0/7168 32bit_api 0 00000080:00200000:1.0:1775764769.955130:0:19723:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08) pos/size18446744073709551614/7168 32bit_api 0 00000080:00200008:1.0:1775764769.955135:0:19723:0:(file.c:447:ll_file_release()) START file ~dmtmp:[0x200000401:0x26c:0x0](ffff8f9952db6c08), flags 304000 00010000:00010000:1.0:1775764769.955142:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971078400/0x36f9bdc9e41ca50c lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ed7b9 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764769.955164:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.955167:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.955170:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:1.0:1775764769.955173:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.955175:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:1.0:1775764769.955178:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764769.955180:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:1.0:1775764769.955183:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764769.955186:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:1.0:1775764769.955188:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764769.955192:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=STUDENTS, flags=1 00000080:00200000:1.0:1775764769.955206:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.955209:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.955211:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=19 00000080:00200000:1.0:1775764769.955214:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.955216:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=19 00000080:00200000:1.0:1775764769.955218:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764769.955220:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=19 00000080:00200000:1.0:1775764769.955221:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764769.955224:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=259 00000080:00200000:1.0:1775764769.955226:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 44 00000080:00200008:1.0:1775764769.955229:0:19723:0:(file.c:1057:ll_file_open()) START file PARADOX:[0x200000401:0x2bb:0x0](ffff8f9952da4b08), flags 304000 00000080:00200008:1.0:1775764769.955237:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file PARADOX:[0x200000401:0x2bb:0x0](ffff8f9952da4b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764769.955240:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08),name=PARADOX 00000002:00010000:1.0:1775764769.955248:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2bb:0x0]) in obj [0x200000401:0x26c:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764769.955256:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c600/0x36f9bdc9e421bcf6 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7b71 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:1.0:1775764769.955263:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c600/0x36f9bdc9e421bcf6 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7b71 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00200000:1.0:1775764769.955270:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26c:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764769.955274:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2bb:0x0](ffff8f9952da4b08) 00000080:00010000:1.0:1775764769.955276:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2bb:0x0](ffff8f9952da4b08) for lock 0x36f9bdc9e421bcf6 00000080:00010000:1.0:1775764769.955280:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421bcf6 from it ffffafbac1c7bcd8 00010000:00010000:1.0:1775764769.955282:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c600/0x36f9bdc9e421bcf6 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f7b71 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:1.0:1775764769.955290:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c600/0x36f9bdc9e421bcf6 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7b71 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00000080:00200000:1.0:1775764769.955302:0:19723:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08) pos/size0/8192 32bit_api 0 00000080:00200000:1.0:1775764769.955321:0:19723:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08) pos/size18446744073709551614/8192 32bit_api 0 00000080:00200008:1.0:1775764769.955327:0:19723:0:(file.c:447:ll_file_release()) START file PARADOX:[0x200000401:0x2bb:0x0](ffff8f9952da4b08), flags 304000 00010000:00010000:1.0:1775764769.955334:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d9c00/0x36f9bdc9e41cab4f lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ee860 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764769.955354:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.955358:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.955360:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:1.0:1775764769.955363:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.955365:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:1.0:1775764769.955368:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764769.955370:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:1.0:1775764769.955372:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764769.955375:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=*, flags=1 00000080:00200000:1.0:1775764769.955385:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.955388:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.955390:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=19 00000080:00200000:1.0:1775764769.955392:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.955394:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=19 00000080:00200000:1.0:1775764769.955396:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764769.955398:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=259 00000080:00200000:1.0:1775764769.955400:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 44 00000080:00200008:1.0:1775764769.955404:0:19723:0:(file.c:1057:ll_file_open()) START file ~dmtmp:[0x200000401:0x26c:0x0](ffff8f9952db6c08), flags 304000 00000080:00200008:1.0:1775764769.955411:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file ~dmtmp:[0x200000401:0x26c:0x0](ffff8f9952db6c08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764769.955414:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08),name=~dmtmp 00000002:00010000:1.0:1775764769.955420:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x26c:0x0]) in obj [0x200000401:0x26a:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764769.955426:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764769.955433:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764769.955439:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26a:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764769.955442:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x26c:0x0](ffff8f9952db6c08) 00000080:00010000:1.0:1775764769.955444:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x26c:0x0](ffff8f9952db6c08) for lock 0x36f9bdc9e41cb2b1 00000080:00010000:1.0:1775764769.955447:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e41cb2b1 from it ffffafbac1c7bcd8 00010000:00010000:1.0:1775764769.955450:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764769.955461:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764769.955474:0:19723:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08) pos/size0/7168 32bit_api 0 00000080:00200000:1.0:1775764769.955485:0:19723:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08) pos/size18446744073709551614/7168 32bit_api 0 00000080:00200008:1.0:1775764769.955490:0:19723:0:(file.c:447:ll_file_release()) START file ~dmtmp:[0x200000401:0x26c:0x0](ffff8f9952db6c08), flags 304000 00010000:00010000:1.0:1775764769.955497:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971078400/0x36f9bdc9e41ca50c lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ed7b9 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764769.955512:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.955515:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.955518:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=19 00000080:00200000:1.0:1775764769.955520:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.955523:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=19 00000080:00200000:1.0:1775764769.955525:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764769.955527:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=259 00000080:00200000:1.0:1775764769.955529:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 44 00000080:00200008:1.0:1775764769.955532:0:19723:0:(file.c:1057:ll_file_open()) START file ~dmtmp:[0x200000401:0x26c:0x0](ffff8f9952db6c08), flags 304000 00000080:00200008:1.0:1775764769.955538:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file ~dmtmp:[0x200000401:0x26c:0x0](ffff8f9952db6c08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764769.955541:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08),name=~dmtmp 00000002:00010000:1.0:1775764769.955547:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x26c:0x0]) in obj [0x200000401:0x26a:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764769.955552:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764769.955559:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764769.955569:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26a:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764769.955572:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x26c:0x0](ffff8f9952db6c08) 00000080:00010000:1.0:1775764769.955574:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x26c:0x0](ffff8f9952db6c08) for lock 0x36f9bdc9e41cb2b1 00000080:00010000:1.0:1775764769.955577:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e41cb2b1 from it ffffafbac1c7bcd8 00010000:00010000:1.0:1775764769.955580:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764769.955587:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764769.955597:0:19723:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08) pos/size0/7168 32bit_api 0 00000080:00200000:1.0:1775764769.955606:0:19723:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08) pos/size18446744073709551614/7168 32bit_api 0 00000080:00200008:1.0:1775764769.955638:0:19723:0:(file.c:447:ll_file_release()) START file ~dmtmp:[0x200000401:0x26c:0x0](ffff8f9952db6c08), flags 304000 00010000:00010000:1.0:1775764769.955647:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971078400/0x36f9bdc9e41ca50c lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ed7b9 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764769.955667:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.955670:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.955673:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:1.0:1775764769.955676:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.955678:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:1.0:1775764769.955680:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764769.955682:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:1.0:1775764769.955684:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764769.955686:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:1.0:1775764769.955688:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764769.955691:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=STUDENTS, flags=1 00000080:00200000:1.0:1775764769.955699:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.955702:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.955704:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=19 00000080:00200000:1.0:1775764769.955706:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.955709:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=19 00000080:00200000:1.0:1775764769.955711:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764769.955713:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=19 00000080:00200000:1.0:1775764769.955715:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764769.955717:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=259 00000080:00200000:1.0:1775764769.955719:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 44 00000080:00200008:1.0:1775764769.955723:0:19723:0:(file.c:1057:ll_file_open()) START file PARADOX:[0x200000401:0x2bb:0x0](ffff8f9952da4b08), flags 304000 00000080:00200008:1.0:1775764769.955729:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file PARADOX:[0x200000401:0x2bb:0x0](ffff8f9952da4b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764769.955732:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08),name=PARADOX 00000002:00010000:1.0:1775764769.955765:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2bb:0x0]) in obj [0x200000401:0x26c:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764769.955771:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c600/0x36f9bdc9e421bcf6 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7b71 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00010000:00010000:1.0:1775764769.955779:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c600/0x36f9bdc9e421bcf6 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7b71 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00000080:00200000:1.0:1775764769.955785:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26c:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764769.955789:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2bb:0x0](ffff8f9952da4b08) 00000080:00010000:1.0:1775764769.955791:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2bb:0x0](ffff8f9952da4b08) for lock 0x36f9bdc9e421bcf6 00000080:00010000:1.0:1775764769.955795:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421bcf6 from it ffffafbac1c7bcd8 00010000:00010000:1.0:1775764769.955797:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c600/0x36f9bdc9e421bcf6 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f7b71 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00010000:00010000:1.0:1775764769.955804:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c600/0x36f9bdc9e421bcf6 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7b71 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00000080:00200000:1.0:1775764769.955817:0:19723:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08) pos/size0/8192 32bit_api 0 00000080:00200000:1.0:1775764769.955830:0:19723:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08) pos/size18446744073709551614/8192 32bit_api 0 00000080:00200008:1.0:1775764769.955836:0:19723:0:(file.c:447:ll_file_release()) START file PARADOX:[0x200000401:0x2bb:0x0](ffff8f9952da4b08), flags 304000 00010000:00010000:1.0:1775764769.955842:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d9c00/0x36f9bdc9e41cab4f lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ee860 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764769.955861:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.955864:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.955867:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:1.0:1775764769.955870:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.955872:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:1.0:1775764769.955874:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764769.955876:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:1.0:1775764769.955879:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764769.955881:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:1.0:1775764769.955883:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764769.955894:0:19723:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=STUDENTS.DB, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), flags=1 00000080:00200000:1.0:1775764769.955899:0:19723:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=STUDENTS.DB, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), intent=getattr 00000002:00010000:1.0:1775764769.955907:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: STUDENTS.DB,[0x0:0x0:0x0]) in obj [0x200000401:0x2bb:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764769.955948:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bec00/0x36f9bdc9e421c0bc lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764769.955954:0:19723:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bec00/0x36f9bdc9e421c0bc lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764769.955963:0:19723:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bec00/0x36f9bdc9e421c0bc lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764769.955989:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970c02d80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966896512:192.168.202.146@tcp:101:dbench.0 00000100:00100000:1.0:1775764769.956083:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f997130da80 going to sleep for 11 seconds 00010000:00010000:0.0:1775764769.957720:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764769.965183:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966896384:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764769.967469:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970c02d80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966896512:192.168.202.146@tcp:101:dbench.0 00010000:00010000:1.0:1775764769.967479:0:19723:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bec00/0x36f9bdc9e421c0bc lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f86df expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764769.967490:0:19723:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bec00/0x36f9bdc9e421c0bc lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2c6:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f86df expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764769.967497:0:19723:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bec00/0x36f9bdc9e421c0bc lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2c6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f86df expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764769.967505:0:19723:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bec00/0x36f9bdc9e421c0bc lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2c6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f86df expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764769.967514:0:19723:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bec00/0x36f9bdc9e421c0bc lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2c6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f86df expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764769.967521:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bec00/0x36f9bdc9e421c0bc lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2c6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f86df expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764769.967528:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bec00/0x36f9bdc9e421c0bc lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2c6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f86df expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764769.967539:0:19723:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bec00/0x36f9bdc9e421c0bc lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2c6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f86df expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764769.967547:0:19723:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970c02d80 x1862022966896512/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/928 e 0 to 0 dl 1775764785 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'dbench.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764769.967555:0:19723:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bec00/0x36f9bdc9e421c0bc lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f86df expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1775764769.967565:0:19723:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bec00/0x36f9bdc9e421c0bc lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f86df expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764769.967573:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970973800/0x36f9bdc9e421be3f lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2c6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f7f45 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764769.967581:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970973800/0x36f9bdc9e421be3f lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2c6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f7f45 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764769.967588:0:19723:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bec00/0x36f9bdc9e421c0bc lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f86df expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764769.967594:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bec00/0x36f9bdc9e421c0bc lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x2aac6a50e5f86df expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764769.967600:0:19723:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bec00/0x36f9bdc9e421c0bc lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x2aac6a50e5f86df expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764769.967640:0:19723:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000080:00200000:1.0:1775764769.967645:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2bb:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1775764769.967698:0:19723:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x2c6:0x0](ffff8f9952e29148): rc = 0 00000080:00200000:1.0:1775764769.967716:0:19723:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x2c6:0x0]: layout version change: 3 -> 3 00000080:00010000:1.0:1775764769.967724:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2c6:0x0](ffff8f9952e29148) for lock 0x36f9bdc9e421be3f 00000080:00010000:1.0:1775764769.967761:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2c6:0x0](ffff8f9952e29148) 00000080:00010000:1.0:1775764769.967764:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421be3f from it ffffafbac1c7bb20 00010000:00010000:1.0:1775764769.967767:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970973800/0x36f9bdc9e421be3f lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f7f45 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764769.967774:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970973800/0x36f9bdc9e421be3f lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f7f45 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200008:1.0:1775764769.967810:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file STUDENTS.DB:[0x200000401:0x2c6:0x0](ffff8f9952e29148), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764769.967815:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2c6:0x0](ffff8f9952e29148),name=STUDENTS.DB 00000002:00010000:1.0:1775764769.967826:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2c6:0x0]) in obj [0x200000401:0x2bb:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764769.967833:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970973800/0x36f9bdc9e421be3f lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2c6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f7f45 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764769.967840:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970973800/0x36f9bdc9e421be3f lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2c6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f7f45 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764769.967846:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2bb:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764769.967850:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2c6:0x0](ffff8f9952e29148) 00000080:00010000:1.0:1775764769.967852:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2c6:0x0](ffff8f9952e29148) for lock 0x36f9bdc9e421be3f 00000080:00010000:1.0:1775764769.967856:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421be3f from it ffffafbac1c7bcb8 00010000:00010000:1.0:1775764769.967859:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970973800/0x36f9bdc9e421be3f lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f7f45 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764769.967865:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970973800/0x36f9bdc9e421be3f lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f7f45 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764769.967888:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2c6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764769.967897:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2c6:0x0] 00000008:00010000:1.0:1775764769.967922:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764769.967925:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99705eeb30, osc lock ffff8f995875c5f0, flags 0x400001002 00010000:00010000:1.0:1775764769.967942:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000094d3ffa7/0x36f9bdc9e41cacb4 lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x59:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eebcb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 177 lru_type: 1 00010000:00010000:1.0:1775764769.967953:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000094d3ffa7/0x36f9bdc9e41cacb4 lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x59:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eebcb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 177 lru_type: 1 00000020:00200000:1.0:1775764769.967964:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764769.967968:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000094d3ffa7/0x36f9bdc9e41cacb4 lrc: 6/0,1 mode: PW/PW res: [0x280000403:0x59:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4eebcb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 177 lru_type: 1 00010000:00010000:1.0:1775764769.967975:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000094d3ffa7/0x36f9bdc9e41cacb4 lrc: 5/0,0 mode: PW/PW res: [0x280000403:0x59:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eebcb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 178 lru_type: 1 00000020:00200000:1.0:1775764769.967984:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764769.967992:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2c6:0x0] updating i_size 260096 i_blocks 515 00000020:00010000:1.0:1775764769.967995:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705f9240 (ffff8f99705b8098/0) at cl_lock_release():245 00000020:00010000:1.0:1775764769.967999:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705f9240 (ffff8f99705b8098/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764769.968001:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705eeb30 (ffff8f99705b87e8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764769.968005:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705f9240 (ffff8f99705b8098/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764769.968008:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705eeb30 (ffff8f99705b8098/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764769.968024:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2c6:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764769.968075:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.968080:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.968085:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:1.0:1775764769.968089:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764769.968093:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:1.0:1775764769.968095:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764769.968098:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:1.0:1775764769.968101:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764769.968104:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:1.0:1775764769.968106:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764769.968116:0:19723:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=__50172.DB, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), flags=1 00000080:00200000:1.0:1775764769.968326:0:19723:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=__50172.DB, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), intent=getattr 00000002:00010000:1.0:1775764769.968338:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: __50172.DB,[0x0:0x0:0x0]) in obj [0x200000401:0x2bb:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764769.968365:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf600/0x36f9bdc9e421c0ca lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764769.968373:0:19723:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf600/0x36f9bdc9e421c0ca lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764769.968380:0:19723:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf600/0x36f9bdc9e421c0ca lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764769.968398:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970c02d80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966897024:192.168.202.146@tcp:101:dbench.0 00000100:00100000:1.0:1775764769.968447:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f997130df00 going to sleep for 11 seconds 00000100:00100000:1.0:1775764769.975818:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764769.975992:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978356096 00000100:00100000:0.0:1775764769.976001:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978356096 00000100:00100000:0.0:1775764769.976052:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24088 00000100:00100000:0.0:1775764769.976061:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970c03b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+5:11866:x1862022978356096:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764769.976077:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d8e00/0x36f9bdc9e421c0b5 lrc: 2/0,0 mode: CW/CW res: [0x200000401:0x2cf:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x2aac6a50e5f86bc expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764769.976157:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764769.976163:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970c03b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+5:11866:x1862022978356096:12345-192.168.202.146@tcp:104: Request processed in 103us (350us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764769.976171:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24088 00010000:00010000:1.0:1775764770.000346:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c600/0x36f9bdc9e421bcf6 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7b71 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00010000:00010000:1.0:1775764770.000364:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c600/0x36f9bdc9e421bcf6 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7b71 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00000080:00010000:1.0:1775764770.000373:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421bcf6 from it ffffafbac1c7b930 00010000:00010000:1.0:1775764770.000378:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c600/0x36f9bdc9e421bcf6 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f7b71 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00010000:00010000:1.0:1775764770.000387:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c600/0x36f9bdc9e421bcf6 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7b71 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00000080:00200000:1.0:1775764770.000490:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.000497:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.000503:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=19 00000080:00200000:1.0:1775764770.000509:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.000512:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=19 00000080:00200000:1.0:1775764770.000515:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.000518:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=19 00000080:00200000:1.0:1775764770.000520:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.000523:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=259 00000080:00200000:1.0:1775764770.000527:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 44 00000080:00200008:1.0:1775764770.000535:0:19723:0:(file.c:1057:ll_file_open()) START file PARADOX:[0x200000401:0x2bb:0x0](ffff8f9952da4b08), flags 304000 00000080:00200008:1.0:1775764770.000552:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file PARADOX:[0x200000401:0x2bb:0x0](ffff8f9952da4b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764770.000557:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08),name=PARADOX 00000002:00010000:1.0:1775764770.000574:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2bb:0x0]) in obj [0x200000401:0x26c:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764770.000581:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c600/0x36f9bdc9e421bcf6 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7b71 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00010000:00010000:1.0:1775764770.000594:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c600/0x36f9bdc9e421bcf6 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7b71 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00000080:00200000:1.0:1775764770.000601:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26c:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764770.000606:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2bb:0x0](ffff8f9952da4b08) 00000080:00010000:1.0:1775764770.000620:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2bb:0x0](ffff8f9952da4b08) for lock 0x36f9bdc9e421bcf6 00000080:00010000:1.0:1775764770.000625:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421bcf6 from it ffffafbac1c7bcd8 00010000:00010000:1.0:1775764770.000635:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c600/0x36f9bdc9e421bcf6 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f7b71 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00010000:00010000:1.0:1775764770.000643:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c600/0x36f9bdc9e421bcf6 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7b71 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 21 lru_type: 1 00000080:00200000:1.0:1775764770.000662:0:19723:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08) pos/size0/8192 32bit_api 0 00000080:00200000:1.0:1775764770.000691:0:19723:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08) pos/size18446744073709551614/8192 32bit_api 0 00000080:00200008:1.0:1775764770.000707:0:19723:0:(file.c:447:ll_file_release()) START file PARADOX:[0x200000401:0x2bb:0x0](ffff8f9952da4b08), flags 304000 00010000:00010000:1.0:1775764770.000717:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d9c00/0x36f9bdc9e41cab4f lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ee860 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764770.000757:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.000761:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.000763:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:1.0:1775764770.000766:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.000769:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:1.0:1775764770.000771:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.000774:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:1.0:1775764770.000776:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.000778:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:1.0:1775764770.000780:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.000783:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=__50172.FAM, flags=257 00000080:00200000:1.0:1775764770.000818:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.000821:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.000823:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:1.0:1775764770.000825:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.000827:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:1.0:1775764770.000830:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.000832:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:1.0:1775764770.000834:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.000836:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:1.0:1775764770.000838:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.000841:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=__50172.DB, flags=1 00000080:00200000:1.0:1775764770.000848:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.000851:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.000854:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=19 00000080:00200000:1.0:1775764770.000856:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.000858:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=19 00000080:00200000:1.0:1775764770.000860:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.000862:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=19 00000080:00200000:1.0:1775764770.000864:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.000866:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=259 00000080:00200000:1.0:1775764770.000869:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 44 00000080:00200008:1.0:1775764770.000872:0:19723:0:(file.c:1057:ll_file_open()) START file PARADOX:[0x200000401:0x2bb:0x0](ffff8f9952da4b08), flags 304000 00000080:00200008:1.0:1775764770.000880:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file PARADOX:[0x200000401:0x2bb:0x0](ffff8f9952da4b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764770.000883:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08),name=PARADOX 00000002:00010000:1.0:1775764770.000890:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2bb:0x0]) in obj [0x200000401:0x26c:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764770.000896:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c600/0x36f9bdc9e421bcf6 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7b71 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 21 lru_type: 1 00010000:00010000:1.0:1775764770.000904:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c600/0x36f9bdc9e421bcf6 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7b71 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 21 lru_type: 1 00000080:00200000:1.0:1775764770.000910:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26c:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764770.000914:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2bb:0x0](ffff8f9952da4b08) 00000080:00010000:1.0:1775764770.000916:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2bb:0x0](ffff8f9952da4b08) for lock 0x36f9bdc9e421bcf6 00000080:00010000:1.0:1775764770.000920:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421bcf6 from it ffffafbac1c7bcd8 00010000:00010000:1.0:1775764770.000922:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c600/0x36f9bdc9e421bcf6 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f7b71 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 21 lru_type: 1 00010000:00010000:1.0:1775764770.000929:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c600/0x36f9bdc9e421bcf6 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7b71 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00000080:00200000:1.0:1775764770.000941:0:19723:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08) pos/size0/8192 32bit_api 0 00000080:00200000:1.0:1775764770.000967:0:19723:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08) pos/size18446744073709551614/8192 32bit_api 0 00000080:00200008:1.0:1775764770.000974:0:19723:0:(file.c:447:ll_file_release()) START file PARADOX:[0x200000401:0x2bb:0x0](ffff8f9952da4b08), flags 304000 00010000:00010000:1.0:1775764770.000981:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d9c00/0x36f9bdc9e41cab4f lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ee860 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764770.000999:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.001002:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.001005:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:1.0:1775764770.001008:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.001010:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:1.0:1775764770.001012:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.001015:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:1.0:1775764770.001017:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.001019:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:1.0:1775764770.001021:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.001023:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=__50172.DB, flags=257 00000080:00200000:1.0:1775764770.001038:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.001041:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.001043:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:1.0:1775764770.001045:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.001047:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:1.0:1775764770.001049:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.001051:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:1.0:1775764770.001053:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.001056:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:1.0:1775764770.001058:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.001060:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=__50172.FAM, flags=1 00000080:00200000:1.0:1775764770.001067:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.001070:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.001072:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=19 00000080:00200000:1.0:1775764770.001074:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.001076:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=19 00000080:00200000:1.0:1775764770.001078:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.001081:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=19 00000080:00200000:1.0:1775764770.001082:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.001085:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=259 00000080:00200000:1.0:1775764770.001087:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 44 00000080:00200008:1.0:1775764770.001090:0:19723:0:(file.c:1057:ll_file_open()) START file PARADOX:[0x200000401:0x2bb:0x0](ffff8f9952da4b08), flags 304000 00000080:00200008:1.0:1775764770.001096:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file PARADOX:[0x200000401:0x2bb:0x0](ffff8f9952da4b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764770.001100:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08),name=PARADOX 00000002:00010000:1.0:1775764770.001106:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2bb:0x0]) in obj [0x200000401:0x26c:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764770.001112:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c600/0x36f9bdc9e421bcf6 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7b71 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00010000:00010000:1.0:1775764770.001488:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c600/0x36f9bdc9e421bcf6 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7b71 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00000080:00200000:1.0:1775764770.001497:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26c:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764770.001503:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2bb:0x0](ffff8f9952da4b08) 00000080:00010000:1.0:1775764770.001507:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2bb:0x0](ffff8f9952da4b08) for lock 0x36f9bdc9e421bcf6 00000080:00010000:1.0:1775764770.001511:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421bcf6 from it ffffafbac1c7bcd8 00010000:00010000:1.0:1775764770.001515:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c600/0x36f9bdc9e421bcf6 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f7b71 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00010000:00010000:1.0:1775764770.001523:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c600/0x36f9bdc9e421bcf6 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7b71 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00000080:00200000:1.0:1775764770.001539:0:19723:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08) pos/size0/8192 32bit_api 0 00000080:00200000:1.0:1775764770.001556:0:19723:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08) pos/size18446744073709551614/8192 32bit_api 0 00000080:00200008:1.0:1775764770.001563:0:19723:0:(file.c:447:ll_file_release()) START file PARADOX:[0x200000401:0x2bb:0x0](ffff8f9952da4b08), flags 304000 00010000:00010000:1.0:1775764770.001571:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d9c00/0x36f9bdc9e41cab4f lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ee860 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764770.001593:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.001597:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.001600:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:1.0:1775764770.001604:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.001606:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:1.0:1775764770.001608:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.001623:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:1.0:1775764770.001633:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.001636:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:1.0:1775764770.001639:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.001641:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=__50172.FAM, flags=257 00000080:00200008:1.0:1775764770.001657:0:19723:0:(file.c:2454:do_file_read_iter()) START file client.txt:[0x200000401:0x262:0x0], ppos: 2547712, count: 8192 00000080:00200000:1.0:1775764770.001687:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.001691:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.001694:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:1.0:1775764770.001696:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.001699:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:1.0:1775764770.001700:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.001703:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:1.0:1775764770.001705:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.001707:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:1.0:1775764770.001709:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.001712:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=__50172.DB, flags=1 00000080:00200000:1.0:1775764770.001721:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.001724:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.001727:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=19 00000080:00200000:1.0:1775764770.001729:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.001731:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=19 00000080:00200000:1.0:1775764770.001733:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.001735:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=19 00000080:00200000:1.0:1775764770.001760:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.001763:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=259 00000080:00200000:1.0:1775764770.001766:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 44 00000080:00200008:1.0:1775764770.001770:0:19723:0:(file.c:1057:ll_file_open()) START file PARADOX:[0x200000401:0x2bb:0x0](ffff8f9952da4b08), flags 304000 00000080:00200008:1.0:1775764770.001778:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file PARADOX:[0x200000401:0x2bb:0x0](ffff8f9952da4b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764770.001782:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08),name=PARADOX 00000002:00010000:1.0:1775764770.001789:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2bb:0x0]) in obj [0x200000401:0x26c:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764770.001796:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c600/0x36f9bdc9e421bcf6 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7b71 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00010000:00010000:1.0:1775764770.001804:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c600/0x36f9bdc9e421bcf6 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7b71 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00000080:00200000:1.0:1775764770.001810:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26c:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764770.001813:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2bb:0x0](ffff8f9952da4b08) 00000080:00010000:1.0:1775764770.001816:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2bb:0x0](ffff8f9952da4b08) for lock 0x36f9bdc9e421bcf6 00000080:00010000:1.0:1775764770.001820:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421bcf6 from it ffffafbac1c7bcd8 00010000:00010000:1.0:1775764770.001822:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c600/0x36f9bdc9e421bcf6 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f7b71 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00010000:00010000:1.0:1775764770.001828:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c600/0x36f9bdc9e421bcf6 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7b71 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00000080:00200000:1.0:1775764770.001839:0:19723:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08) pos/size0/8192 32bit_api 0 00000080:00200000:1.0:1775764770.001852:0:19723:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08) pos/size18446744073709551614/8192 32bit_api 0 00000080:00200008:1.0:1775764770.001858:0:19723:0:(file.c:447:ll_file_release()) START file PARADOX:[0x200000401:0x2bb:0x0](ffff8f9952da4b08), flags 304000 00010000:00010000:1.0:1775764770.001864:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d9c00/0x36f9bdc9e41cab4f lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ee860 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764770.001879:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.001882:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.001884:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:1.0:1775764770.001887:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.001890:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:1.0:1775764770.001892:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.001895:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:1.0:1775764770.001897:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.001900:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:1.0:1775764770.001902:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.001907:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=__50172.DB, flags=769 00000080:00200000:1.0:1775764770.001909:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 3 00000080:00200000:1.0:1775764770.001913:0:19723:0:(namei.c:1399:ll_atomic_open()) VFS Op:name=__50172.DB, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), file ffff8f9945fdec00, open_flags 8042, mode 8180 opened 0 00000080:00200000:1.0:1775764770.001921:0:19723:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=__50172.DB, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), intent=open|creat 00000002:00010000:1.0:1775764770.001935:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: __50172.DB,[0x200000401:0xed6:0x0]) in obj [0x200000401:0x2bb:0x0], intent: open|creat flags 0100103 00010000:00010000:1.0:1775764770.001947:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c600/0x36f9bdc9e421bcf6 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f7b71 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00010000:00010000:1.0:1775764770.003540:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d9c00/0x36f9bdc9e41cab4f lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x5/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ee860 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764770.003550:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970adec00/0x36f9bdc9e41cab8e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x11/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ee908 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764770.003684:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c600/0x36f9bdc9e421bcf6 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f7b71 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00010000:00010000:1.0:1775764770.003692:0:19723:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764770.003695:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c600/0x36f9bdc9e421bcf6 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f7b71 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00010000:00010000:1.0:1775764770.003716:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bea00/0x36f9bdc9e421c0df lrc: 3/0,1 mode: --/CW res: [0x200000401:0x2bb:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.003723:0:19723:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bea00/0x36f9bdc9e421c0df lrc: 3/0,1 mode: --/CW res: [0x200000401:0x2bb:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764770.003733:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764770.003759:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970c01880 x1862022966898176/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4624/70288 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764770.003769:0:19723:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bea00/0x36f9bdc9e421c0df lrc: 3/0,1 mode: --/CW res: [0x200000401:0x2bb:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764770.003790:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970c01880 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966898176:192.168.202.146@tcp:101:dbench.0 00000100:00100000:1.0:1775764770.003912:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f997130d000 going to sleep for 11 seconds 00000080:00200000:1.0:1775764770.012727:0:36:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2b9:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764770.012756:0:36:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:1.0:1775764770.012777:0:36:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:1.0:1775764770.012779:0:36:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:1.0:1775764770.012800:0:36:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2b9:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:1.0:1775764770.016877:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994eeac700 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966898304:192.168.202.146@tcp:101:dbench.0 00010000:00010000:1.0:1775764770.016898:0:19724:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d8e00/0x36f9bdc9e421c0e6 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.016912:0:19724:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d8e00/0x36f9bdc9e421c0e6 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x206400000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.016919:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d8e00/0x36f9bdc9e421c0e6 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x10206400000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.016926:0:19724:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d8e00/0x36f9bdc9e421c0e6 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x10206400000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.016934:0:19724:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d8e00/0x36f9bdc9e421c0e6 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x206400000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.016945:0:19724:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f994f3d8e00 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:1.0:1775764770.016951:0:19724:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d8e00/0x36f9bdc9e421c0e6 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x2/0x2 rrc: 5 type: IBT flags: 0x206400000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.016967:0:19724:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d8e00/0x36f9bdc9e421c0e6 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x2/0x2 rrc: 5 type: IBT flags: 0x206400000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.016974:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d8e00/0x36f9bdc9e421c0e6 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x2/0x2 rrc: 5 type: IBT flags: 0x20e400000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.016981:0:19724:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1775764770.016987:0:19724:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d8e00/0x36f9bdc9e421c0e6 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x2/0x2 rrc: 5 type: IBT flags: 0x4a0f400000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.016993:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d8e00/0x36f9bdc9e421c0e6 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x2/0x2 rrc: 5 type: IBT flags: 0x4a0f400000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764770.017011:0:19724:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=7, status=0 req@ffff8f994eeac700 x1862022966898304/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/584 e 0 to 0 dl 1775764786 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'dbench.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764770.017025:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2b8:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1775764770.017051:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421be70 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7fed expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00010000:00010000:1.0:1775764770.017064:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421be70 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7fed expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00000080:00010000:1.0:1775764770.017073:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421be70 from it ffffafbac1c5b930 00010000:00010000:1.0:1775764770.017077:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421be70 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f7fed expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00010000:00010000:1.0:1775764770.017091:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421be70 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7fed expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00000080:00200000:1.0:1775764770.017186:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.017193:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.017198:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=19 00000080:00200000:1.0:1775764770.017205:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.017208:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=19 00000080:00200000:1.0:1775764770.017211:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.017214:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=19 00000080:00200000:1.0:1775764770.017216:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.017220:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=259 00000080:00200000:1.0:1775764770.017224:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 44 00000080:00200008:1.0:1775764770.017232:0:19724:0:(file.c:1057:ll_file_open()) START file PARADOX:[0x200000401:0x2b8:0x0](ffff8f9952d880c8), flags 304000 00000080:00200008:1.0:1775764770.017250:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file PARADOX:[0x200000401:0x2b8:0x0](ffff8f9952d880c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764770.017255:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8),name=PARADOX 00000002:00010000:1.0:1775764770.017271:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2b8:0x0]) in obj [0x200000401:0x26b:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764770.017279:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421be70 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7fed expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00010000:00010000:1.0:1775764770.017287:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421be70 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7fed expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00000080:00200000:1.0:1775764770.017295:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26b:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764770.017300:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2b8:0x0](ffff8f9952d880c8) 00000080:00010000:1.0:1775764770.017303:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2b8:0x0](ffff8f9952d880c8) for lock 0x36f9bdc9e421be70 00000080:00010000:1.0:1775764770.017307:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421be70 from it ffffafbac1c5bcd8 00010000:00010000:1.0:1775764770.017322:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421be70 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f7fed expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00010000:00010000:1.0:1775764770.017330:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421be70 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7fed expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00000080:00200000:1.0:1775764770.017349:0:19724:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8) pos/size0/8192 32bit_api 0 00000080:00200000:1.0:1775764770.017378:0:19724:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8) pos/size18446744073709551614/8192 32bit_api 0 00000080:00200008:1.0:1775764770.017391:0:19724:0:(file.c:447:ll_file_release()) START file PARADOX:[0x200000401:0x2b8:0x0](ffff8f9952d880c8), flags 304000 00010000:00010000:1.0:1775764770.017564:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e41caad1 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ee710 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764770.017677:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.017681:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.017684:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:1.0:1775764770.017688:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.017691:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:1.0:1775764770.017693:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.017696:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:1.0:1775764770.017698:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.017700:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:1.0:1775764770.017702:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.017714:0:19724:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=__52E2B.FAM, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), flags=1 00000080:00200000:1.0:1775764770.017718:0:19724:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=__52E2B.FAM, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), intent=getattr 00000002:00010000:1.0:1775764770.017727:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: __52E2B.FAM,[0x0:0x0:0x0]) in obj [0x200000401:0x2b8:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764770.017787:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be400/0x36f9bdc9e421c0ed lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.017795:0:19724:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be400/0x36f9bdc9e421c0ed lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.017805:0:19724:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be400/0x36f9bdc9e421c0ed lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764770.017825:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970c00000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966898432:192.168.202.146@tcp:101:dbench.0 00000100:00100000:1.0:1775764770.017905:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f997130db40 going to sleep for 11 seconds 00000100:00100000:1.0:1775764770.022022:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970c00000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966898432:192.168.202.146@tcp:101:dbench.0 00010000:00010000:1.0:1775764770.022045:0:19724:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be400/0x36f9bdc9e421c0ed lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.022060:0:19724:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be400/0x36f9bdc9e421c0ed lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x206400000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.022068:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be400/0x36f9bdc9e421c0ed lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x10206400000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.022076:0:19724:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be400/0x36f9bdc9e421c0ed lrc: 3/0,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x10206400000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.022085:0:19724:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be400/0x36f9bdc9e421c0ed lrc: 4/0,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x206400000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.022091:0:19724:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99668be400 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:1.0:1775764770.022096:0:19724:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be400/0x36f9bdc9e421c0ed lrc: 4/0,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x2/0x2 rrc: 5 type: IBT flags: 0x206400000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.022104:0:19724:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be400/0x36f9bdc9e421c0ed lrc: 4/0,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x2/0x2 rrc: 5 type: IBT flags: 0x206400000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.022112:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be400/0x36f9bdc9e421c0ed lrc: 5/0,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x2/0x2 rrc: 5 type: IBT flags: 0x20e400000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.022546:0:19724:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1775764770.022554:0:19724:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be400/0x36f9bdc9e421c0ed lrc: 3/0,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x2/0x2 rrc: 5 type: IBT flags: 0x4a0f400000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.022562:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be400/0x36f9bdc9e421c0ed lrc: 0/0,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x2/0x2 rrc: 5 type: IBT flags: 0x4a0f400000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764770.022581:0:19724:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=7, status=0 req@ffff8f9970c00000 x1862022966898432/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/584 e 0 to 0 dl 1775764786 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'dbench.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764770.022595:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2b8:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1775764770.022643:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421be70 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7fed expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00010000:00010000:1.0:1775764770.022655:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421be70 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7fed expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00000080:00010000:1.0:1775764770.022664:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421be70 from it ffffafbac1c5b930 00010000:00010000:1.0:1775764770.022667:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421be70 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f7fed expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00010000:00010000:1.0:1775764770.022675:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421be70 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7fed expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00000080:00200000:1.0:1775764770.022775:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.022782:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.022788:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=19 00000080:00200000:1.0:1775764770.022794:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.022798:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=19 00000080:00200000:1.0:1775764770.022801:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.022804:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=19 00000080:00200000:1.0:1775764770.022807:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.022811:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=259 00000080:00200000:1.0:1775764770.022815:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 44 00000080:00200008:1.0:1775764770.022822:0:19724:0:(file.c:1057:ll_file_open()) START file PARADOX:[0x200000401:0x2b8:0x0](ffff8f9952d880c8), flags 304000 00000080:00200008:1.0:1775764770.022842:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file PARADOX:[0x200000401:0x2b8:0x0](ffff8f9952d880c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764770.022847:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8),name=PARADOX 00000002:00010000:1.0:1775764770.022863:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2b8:0x0]) in obj [0x200000401:0x26b:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764770.022871:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421be70 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7fed expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00010000:00010000:1.0:1775764770.022879:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421be70 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7fed expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00000080:00200000:1.0:1775764770.022886:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26b:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764770.022890:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2b8:0x0](ffff8f9952d880c8) 00000080:00010000:1.0:1775764770.022893:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2b8:0x0](ffff8f9952d880c8) for lock 0x36f9bdc9e421be70 00000080:00010000:1.0:1775764770.022898:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421be70 from it ffffafbac1c5bcd8 00010000:00010000:1.0:1775764770.022900:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421be70 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f7fed expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00010000:00010000:1.0:1775764770.022908:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421be70 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7fed expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:1.0:1775764770.022929:0:19724:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8) pos/size0/8192 32bit_api 0 00000080:00200000:1.0:1775764770.022974:0:19724:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8) pos/size18446744073709551614/8192 32bit_api 0 00000080:00200008:1.0:1775764770.022988:0:19724:0:(file.c:447:ll_file_release()) START file PARADOX:[0x200000401:0x2b8:0x0](ffff8f9952d880c8), flags 304000 00010000:00010000:1.0:1775764770.022999:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e41caad1 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ee710 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764770.023019:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.023023:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.023026:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:1.0:1775764770.023029:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.023032:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:1.0:1775764770.023034:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.023037:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:1.0:1775764770.023039:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.023042:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:1.0:1775764770.023045:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.023048:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=__52E2B.FAM, flags=257 00000080:00200000:1.0:1775764770.023081:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.023084:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.023087:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:1.0:1775764770.023090:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.023092:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:1.0:1775764770.023095:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.023098:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:1.0:1775764770.023100:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.023102:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:1.0:1775764770.023105:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.023108:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=__52E2B.DB, flags=1 00000080:00200000:1.0:1775764770.023116:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.025856:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.025864:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=19 00000080:00200000:1.0:1775764770.025873:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.025877:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=19 00000080:00200000:1.0:1775764770.025880:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.025883:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=19 00000080:00200000:1.0:1775764770.025886:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.025890:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=259 00000080:00200000:1.0:1775764770.025894:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 44 00000080:00200008:1.0:1775764770.025902:0:19724:0:(file.c:1057:ll_file_open()) START file PARADOX:[0x200000401:0x2b8:0x0](ffff8f9952d880c8), flags 304000 00000080:00200008:1.0:1775764770.025927:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file PARADOX:[0x200000401:0x2b8:0x0](ffff8f9952d880c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764770.025932:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8),name=PARADOX 00000002:00010000:1.0:1775764770.025947:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2b8:0x0]) in obj [0x200000401:0x26b:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764770.026219:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421be70 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7fed expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:1.0:1775764770.026230:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421be70 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7fed expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:1.0:1775764770.026237:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26b:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764770.026242:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2b8:0x0](ffff8f9952d880c8) 00000080:00010000:1.0:1775764770.026245:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2b8:0x0](ffff8f9952d880c8) for lock 0x36f9bdc9e421be70 00000080:00010000:1.0:1775764770.026249:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421be70 from it ffffafbac1c5bcd8 00010000:00010000:1.0:1775764770.026252:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421be70 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f7fed expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:1.0:1775764770.026260:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421be70 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7fed expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:1.0:1775764770.026283:0:19724:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8) pos/size0/8192 32bit_api 0 00000080:00200000:1.0:1775764770.026312:0:19724:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8) pos/size18446744073709551614/8192 32bit_api 0 00000080:00200008:1.0:1775764770.026328:0:19724:0:(file.c:447:ll_file_release()) START file PARADOX:[0x200000401:0x2b8:0x0](ffff8f9952d880c8), flags 304000 00010000:00010000:1.0:1775764770.026340:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e41caad1 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ee710 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764770.026374:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.026378:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.026381:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:1.0:1775764770.026385:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.026387:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:1.0:1775764770.026389:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.026941:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:1.0:1775764770.026946:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.026958:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:1.0:1775764770.026961:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.026965:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=__52E2B.DB, flags=257 00000080:00200000:1.0:1775764770.027016:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.027019:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.027022:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:1.0:1775764770.027025:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.027028:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:1.0:1775764770.027030:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.027033:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:1.0:1775764770.027035:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.027037:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:1.0:1775764770.027039:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.027042:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=__52E2B.FAM, flags=1 00000080:00200000:1.0:1775764770.027053:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.027056:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.027059:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=19 00000080:00200000:1.0:1775764770.027061:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.027063:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=19 00000080:00200000:1.0:1775764770.027066:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.027068:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=19 00000080:00200000:1.0:1775764770.027070:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.027072:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=259 00000080:00200000:1.0:1775764770.027075:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 44 00000080:00200008:1.0:1775764770.027080:0:19724:0:(file.c:1057:ll_file_open()) START file PARADOX:[0x200000401:0x2b8:0x0](ffff8f9952d880c8), flags 304000 00000080:00200008:1.0:1775764770.027090:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file PARADOX:[0x200000401:0x2b8:0x0](ffff8f9952d880c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764770.027094:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8),name=PARADOX 00000002:00010000:1.0:1775764770.027104:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2b8:0x0]) in obj [0x200000401:0x26b:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764770.027113:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421be70 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7fed expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:1.0:1775764770.027132:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421be70 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7fed expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:1.0:1775764770.027139:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26b:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764770.027143:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2b8:0x0](ffff8f9952d880c8) 00000080:00010000:1.0:1775764770.027146:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2b8:0x0](ffff8f9952d880c8) for lock 0x36f9bdc9e421be70 00000080:00010000:1.0:1775764770.027150:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421be70 from it ffffafbac1c5bcd8 00010000:00010000:1.0:1775764770.027152:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421be70 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f7fed expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:1.0:1775764770.027160:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421be70 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7fed expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:1.0:1775764770.027174:0:19724:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8) pos/size0/8192 32bit_api 0 00000080:00200000:1.0:1775764770.027192:0:19724:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8) pos/size18446744073709551614/8192 32bit_api 0 00000080:00200008:1.0:1775764770.027199:0:19724:0:(file.c:447:ll_file_release()) START file PARADOX:[0x200000401:0x2b8:0x0](ffff8f9952d880c8), flags 304000 00010000:00010000:1.0:1775764770.027207:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e41caad1 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ee710 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764770.027223:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.027226:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.027229:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:1.0:1775764770.027232:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.027234:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:1.0:1775764770.027236:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.027239:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:1.0:1775764770.027241:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.027244:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:1.0:1775764770.027246:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.027248:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=__52E2B.FAM, flags=257 00000080:00200000:1.0:1775764770.027261:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.027264:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.027266:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:1.0:1775764770.027269:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.027272:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:1.0:1775764770.027274:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.027276:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:1.0:1775764770.027278:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.027281:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:1.0:1775764770.027283:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.027285:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=__52E2B.DB, flags=1 00000080:00200000:1.0:1775764770.027293:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.027296:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.027299:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=19 00000080:00200000:1.0:1775764770.027301:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.027303:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=19 00000080:00200000:1.0:1775764770.027305:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.027308:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=19 00000080:00200000:1.0:1775764770.027310:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.027312:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=259 00000080:00200000:1.0:1775764770.027314:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 44 00000080:00200008:1.0:1775764770.027318:0:19724:0:(file.c:1057:ll_file_open()) START file PARADOX:[0x200000401:0x2b8:0x0](ffff8f9952d880c8), flags 304000 00000080:00200008:1.0:1775764770.027324:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file PARADOX:[0x200000401:0x2b8:0x0](ffff8f9952d880c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764770.027328:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8),name=PARADOX 00000002:00010000:1.0:1775764770.027335:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2b8:0x0]) in obj [0x200000401:0x26b:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764770.027342:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421be70 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7fed expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:1.0:1775764770.027350:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421be70 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7fed expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:1.0:1775764770.027356:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26b:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764770.027360:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2b8:0x0](ffff8f9952d880c8) 00000080:00010000:1.0:1775764770.027362:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2b8:0x0](ffff8f9952d880c8) for lock 0x36f9bdc9e421be70 00000080:00010000:1.0:1775764770.027365:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421be70 from it ffffafbac1c5bcd8 00010000:00010000:1.0:1775764770.027368:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421be70 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f7fed expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:1.0:1775764770.027375:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421be70 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f7fed expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:1.0:1775764770.027388:0:19724:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8) pos/size0/8192 32bit_api 0 00000080:00200000:1.0:1775764770.027417:0:19724:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8) pos/size18446744073709551614/8192 32bit_api 0 00000080:00200008:1.0:1775764770.027424:0:19724:0:(file.c:447:ll_file_release()) START file PARADOX:[0x200000401:0x2b8:0x0](ffff8f9952d880c8), flags 304000 00010000:00010000:1.0:1775764770.027432:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e41caad1 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ee710 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764770.027450:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.027453:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.027456:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:1.0:1775764770.027459:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.027462:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:1.0:1775764770.027465:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.027467:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:1.0:1775764770.027470:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.027473:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:1.0:1775764770.027475:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.027479:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=__52E2B.DB, flags=769 00000080:00200000:1.0:1775764770.027482:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 3 00000080:00200000:1.0:1775764770.027486:0:19724:0:(namei.c:1399:ll_atomic_open()) VFS Op:name=__52E2B.DB, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), file ffff8f9945fdfc80, open_flags 8042, mode 8180 opened 0 00000080:00200000:1.0:1775764770.027495:0:19724:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=__52E2B.DB, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), intent=open|creat 00000002:00010000:1.0:1775764770.027509:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: __52E2B.DB,[0x200000401:0xed7:0x0]) in obj [0x200000401:0x2b8:0x0], intent: open|creat flags 0100103 00010000:00010000:1.0:1775764770.027520:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421be70 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f7fed expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:1.0:1775764770.027534:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e41caad1 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x5/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ee710 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764770.027545:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970abee00/0x36f9bdc9e41cab09 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x11/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ee795 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 78 lru_type: 1 00010000:00010000:1.0:1775764770.027643:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421be70 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f7fed expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:1.0:1775764770.027655:0:19724:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764770.027657:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421be70 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f7fed expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:1.0:1775764770.027678:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bfc00/0x36f9bdc9e421c0fb lrc: 3/0,1 mode: --/CW res: [0x200000401:0x2b8:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.027684:0:19724:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bfc00/0x36f9bdc9e421c0fb lrc: 3/0,1 mode: --/CW res: [0x200000401:0x2b8:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764770.027695:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764770.027702:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970c00000 x1862022966898816/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4624/70288 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764770.027711:0:19724:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bfc00/0x36f9bdc9e421c0fb lrc: 3/0,1 mode: --/CW res: [0x200000401:0x2b8:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764770.027727:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970c00000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966898816:192.168.202.146@tcp:101:dbench.0 00000100:00100000:1.0:1775764770.044723:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99707ef540 going to sleep for 11 seconds 00000100:00100000:1.0:1775764770.057849:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703dd500 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966899072:192.168.202.146@tcp:35:dbench.0 00000002:00100000:1.0:1775764770.057881:0:19723:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f9970c01880 x1862022966898176/t4294997328(4294997328) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 872/952 e 0 to 0 dl 1775764786 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764770.057969:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.057976:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.057981:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:1.0:1775764770.057987:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.057990:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:1.0:1775764770.057993:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.057996:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:1.0:1775764770.057998:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.058001:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:1.0:1775764770.058003:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.058006:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=__50172.DB, flags=1 00000080:00200008:1.0:1775764770.058012:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file __50172.DB:[0x200000401:0xed6:0x0](ffff8f9952ee9148), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764770.058017:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xed6:0x0](ffff8f9952ee9148),name=__50172.DB 00000002:00010000:1.0:1775764770.058037:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xed6:0x0]) in obj [0x200000401:0x2bb:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764770.058617:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bea00/0x36f9bdc9e421c0df lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f875d expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764770.058645:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bea00/0x36f9bdc9e421c0df lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f875d expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764770.058655:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2bb:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764770.058662:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xed6:0x0](ffff8f9952ee9148) 00000080:00010000:1.0:1775764770.058665:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xed6:0x0](ffff8f9952ee9148) for lock 0x36f9bdc9e421c0df 00000080:00010000:1.0:1775764770.058672:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c0df from it ffffafbac1c7bcb8 00010000:00010000:1.0:1775764770.058676:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bea00/0x36f9bdc9e421c0df lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f875d expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764770.058685:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bea00/0x36f9bdc9e421c0df lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f875d expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764770.058787:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xed6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764770.058799:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xed6:0x0] 00000008:00010000:1.0:1775764770.058826:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764770.058829:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99705eeb30, osc lock ffff8f995875c5f0, flags 0x400001002 00010000:00010000:1.0:1775764770.058848:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000aa9eb34d/0x36f9bdc9e421c0f4 lrc: 2/1,0 mode: PR/PR res: [0x280000403:0x65f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5f8780 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.058861:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000aa9eb34d/0x36f9bdc9e421c0f4 lrc: 2/1,0 mode: PR/PR res: [0x280000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5f8780 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:1.0:1775764770.058873:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764770.058878:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000aa9eb34d/0x36f9bdc9e421c0f4 lrc: 5/1,0 mode: PR/PR res: [0x280000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5f8780 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.058886:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000aa9eb34d/0x36f9bdc9e421c0f4 lrc: 4/0,0 mode: PR/PR res: [0x280000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5f8780 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:1.0:1775764770.058896:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764770.058906:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0xed6:0x0] updating i_size 0 i_blocks 1 00000020:00010000:1.0:1775764770.058910:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705f9240 (ffff8f99705b8098/0) at cl_lock_release():245 00000020:00010000:1.0:1775764770.058913:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705f9240 (ffff8f99705b8098/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764770.058916:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705eeb30 (ffff8f99705b87e8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764770.058920:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705f9240 (ffff8f99705b8098/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764770.058923:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705eeb30 (ffff8f99705b8098/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764770.058957:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xed6:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764770.058994:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.058999:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.059004:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:1.0:1775764770.059009:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.059012:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:1.0:1775764770.059014:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.059018:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:1.0:1775764770.059021:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.059024:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:1.0:1775764770.059893:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.059900:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=__50172.DB, flags=257 00000080:00200000:1.0:1775764770.059904:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xed6:0x0](ffff8f9952ee9148), inode mode 8180 mask 46 00000080:00200008:1.0:1775764770.059912:0:19723:0:(file.c:1057:ll_file_open()) START file __50172.DB:[0x200000401:0xed6:0x0](ffff8f9952ee9148), flags 100002 00000002:00010000:1.0:1775764770.059931:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xed6:0x0]) in obj [0x200000401:0x2bb:0x0], intent: open flags 04240100003 00010000:00010000:1.0:1775764770.060044:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c109 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.060052:0:19723:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c109 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764770.060062:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764770.060070:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970c03480 x1862022966899456/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/135824 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764770.060078:0:19723:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c109 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764770.060097:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970c03480 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966899456:192.168.202.146@tcp:101:dbench.0 00000080:00200000:0.0:1775764770.060388:0:19724:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0xed7:0x0] updating i_size 0 i_blocks 1 00000020:00010000:0.0:1775764770.060401:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970459ff0 (ffff8f994ef58f38/0) at cl_lock_release():245 00000020:00010000:0.0:1775764770.060404:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970459ff0 (ffff8f994ef58f38/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.060408:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970d8d030 (ffff8f994ef58998/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.060412:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970459ff0 (ffff8f994ef58f38/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764770.060415:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970d8d030 (ffff8f994ef58f38/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764770.060443:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xed7:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764770.060496:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.060500:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.060504:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:0.0:1775764770.060508:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.060511:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:0.0:1775764770.060514:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.060517:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:0.0:1775764770.060519:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.060523:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=*, flags=1 00000080:00200000:0.0:1775764770.060539:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.060542:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.060545:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=19 00000080:00200000:0.0:1775764770.060547:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.060549:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=19 00000080:00200000:0.0:1775764770.060551:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.060554:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=259 00000080:00200000:0.0:1775764770.060556:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 44 00000080:00200008:0.0:1775764770.060561:0:19724:0:(file.c:1057:ll_file_open()) START file ~dmtmp:[0x200000401:0x26b:0x0](ffff8f9952db00c8), flags 304000 00000080:00200008:0.0:1775764770.060574:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file ~dmtmp:[0x200000401:0x26b:0x0](ffff8f9952db00c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764770.060579:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8),name=~dmtmp 00000002:00010000:0.0:1775764770.060591:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x26b:0x0]) in obj [0x200000401:0x269:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764770.060601:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9800/0x36f9bdc9e41cb202 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ef946 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764770.060611:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9800/0x36f9bdc9e41cb202 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ef946 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764770.060619:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x269:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764770.060624:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x26b:0x0](ffff8f9952db00c8) 00000080:00010000:0.0:1775764770.060639:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x26b:0x0](ffff8f9952db00c8) for lock 0x36f9bdc9e41cb202 00000080:00010000:0.0:1775764770.060643:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e41cb202 from it ffffafbac1c5bcd8 00010000:00010000:0.0:1775764770.060646:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9800/0x36f9bdc9e41cb202 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e4ef946 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764770.060654:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9800/0x36f9bdc9e41cb202 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ef946 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764770.060672:0:19724:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8) pos/size0/7168 32bit_api 0 00000080:00200000:0.0:1775764770.060697:0:19724:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8) pos/size18446744073709551614/7168 32bit_api 0 00000080:00200008:0.0:1775764770.060710:0:19724:0:(file.c:447:ll_file_release()) START file ~dmtmp:[0x200000401:0x26b:0x0](ffff8f9952db00c8), flags 304000 00010000:00010000:0.0:1775764770.060720:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9940030600/0x36f9bdc9e41ca521 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ed7e3 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764770.060779:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.060783:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.060786:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=19 00000080:00200000:0.0:1775764770.060789:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.060792:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=19 00000080:00200000:0.0:1775764770.060794:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.060796:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=259 00000080:00200000:0.0:1775764770.060799:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 44 00000080:00200008:0.0:1775764770.060803:0:19724:0:(file.c:1057:ll_file_open()) START file ~dmtmp:[0x200000401:0x26b:0x0](ffff8f9952db00c8), flags 304000 00000080:00200008:0.0:1775764770.060810:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file ~dmtmp:[0x200000401:0x26b:0x0](ffff8f9952db00c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764770.060814:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8),name=~dmtmp 00000002:00010000:0.0:1775764770.060821:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x26b:0x0]) in obj [0x200000401:0x269:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764770.060827:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9800/0x36f9bdc9e41cb202 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ef946 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764770.060835:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9800/0x36f9bdc9e41cb202 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ef946 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764770.060841:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x269:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764770.060844:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x26b:0x0](ffff8f9952db00c8) 00000080:00010000:0.0:1775764770.060847:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x26b:0x0](ffff8f9952db00c8) for lock 0x36f9bdc9e41cb202 00000080:00010000:0.0:1775764770.060850:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e41cb202 from it ffffafbac1c5bcd8 00010000:00010000:0.0:1775764770.060853:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9800/0x36f9bdc9e41cb202 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e4ef946 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764770.060859:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9800/0x36f9bdc9e41cb202 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ef946 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764770.060870:0:19724:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8) pos/size0/7168 32bit_api 0 00000080:00200000:0.0:1775764770.060881:0:19724:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8) pos/size18446744073709551614/7168 32bit_api 0 00000080:00200008:0.0:1775764770.060887:0:19724:0:(file.c:447:ll_file_release()) START file ~dmtmp:[0x200000401:0x26b:0x0](ffff8f9952db00c8), flags 304000 00010000:00010000:0.0:1775764770.060894:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9940030600/0x36f9bdc9e41ca521 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ed7e3 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200008:0.0:1775764770.060915:0:19724:0:(file.c:447:ll_file_release()) START file __52E2B.DB:[0x200000401:0xed7:0x0](ffff8f9952eb4b08), flags 100002 00000002:00100000:0.0:1775764770.060947:0:19724:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f9970c00000 x1862022966898816/t4294997329(4294997329) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 872/952 e 0 to 0 dl 1775764786 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1775764770.060969:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 2 is allocated opc 35, max 7 00000100:00100000:0.0:1775764770.060973:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e4000 x1862022966899712/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1775764770.060988:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966899712:192.168.202.146@tcp:35:dbench.0 00000100:00100000:1.0:1775764770.061638:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f997130de40 going to sleep for 11 seconds 00000100:00100000:0.0:1775764770.064193:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f180 going to sleep for 11 seconds 00000100:00100000:1.0:1775764770.069115:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764770.073898:0:11146:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978356224 00000100:00100000:0.0:1775764770.073912:0:11146:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978356224 00000100:00100000:0.0:1775764770.074070:0:11146:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24089 00000100:00100000:0.0:1775764770.074080:0:11146:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970c02d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:3f83546c-01a9-4156-8a97-44c446245d3e+5:11866:x1862022978356224:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764770.074096:0:11146:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bea00/0x36f9bdc9e421c0df lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x2aac6a50e5f875d expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764770.074654:0:11146:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764770.074663:0:11146:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970c02d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:3f83546c-01a9-4156-8a97-44c446245d3e+5:11866:x1862022978356224:12345-192.168.202.146@tcp:104: Request processed in 584us (5556us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764770.074674:0:11146:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24089 00000100:00100000:1.0:1775764770.082947:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970c03480 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966899456:192.168.202.146@tcp:101:dbench.0 00010000:00010000:1.0:1775764770.082985:0:19723:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode CW ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c109 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f87b8 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.083005:0:19723:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c109 lrc: 4/1,0 mode: --/CW res: [0x200000401:0xed6:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f87b8 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.083013:0:19723:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c109 lrc: 4/1,0 mode: --/CW res: [0x200000401:0xed6:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f87b8 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.083023:0:19723:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c109 lrc: 4/1,0 mode: CW/CW res: [0x200000401:0xed6:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f87b8 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.083032:0:19723:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c109 lrc: 4/1,0 mode: CW/CW res: [0x200000401:0xed6:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f87b8 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.083045:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c109 lrc: 5/1,1 mode: CW/CW res: [0x200000401:0xed6:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f87b8 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.083052:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c109 lrc: 5/1,1 mode: CW/CW res: [0x200000401:0xed6:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f87b8 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.083059:0:19723:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c109 lrc: 4/0,1 mode: CW/CW res: [0x200000401:0xed6:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f87b8 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764770.083068:0:19723:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=200002b, status=0 req@ffff8f9970c03480 x1862022966899456/t4294997332(4294997332) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/952 e 0 to 0 dl 1775764786 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dbench.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764770.083079:0:19723:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f9970c03480 x1862022966899456/t4294997332(4294997332) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/952 e 0 to 0 dl 1775764786 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dbench.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764770.083092:0:19723:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c109 lrc: 3/0,1 mode: CW/CW res: [0x200000401:0xed6:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f87b8 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.083101:0:19723:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 8 res 8589935617/3798 (0 0) 00000080:00200000:1.0:1775764770.083109:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0x2bb:0x0] suppgids 0 0: rc 0 00000080:00010000:1.0:1775764770.083204:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xed6:0x0](ffff8f9952ee9148) for lock 0x36f9bdc9e421c109 00000080:00010000:1.0:1775764770.083212:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c109 from it ffffafbac1c7bbf0 00010000:00010000:1.0:1775764770.083215:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c109 lrc: 3/0,1 mode: CW/CW res: [0x200000401:0xed6:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f87b8 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.083224:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c109 lrc: 2/0,0 mode: CW/CW res: [0x200000401:0xed6:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f87b8 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000002:00100000:1.0:1775764770.083247:0:19723:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f9970c03480 x1862022966899456/t4294997332(4294997332) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/952 e 0 to 0 dl 1775764786 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dbench.0' uid:0 gid:0 projid:0 00000080:00200008:1.0:1775764770.083326:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file __50172.DB:[0x200000401:0xed6:0x0](ffff8f9952ee9148), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764770.083333:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xed6:0x0](ffff8f9952ee9148),name=__50172.DB 00000002:00010000:1.0:1775764770.083346:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xed6:0x0]) in obj [0x200000401:0xed6:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764770.083354:0:19723:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/3798 (0 0) 00010000:00010000:1.0:1775764770.083389:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c117 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xed6:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.083408:0:19723:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c117 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xed6:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.083418:0:19723:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c117 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xed6:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764770.083437:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970c03100 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966900224:192.168.202.146@tcp:101:dbench.0 00000100:00100000:1.0:1775764770.083545:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f997130de40 going to sleep for 11 seconds 00000100:00100000:0.0:1775764770.086853:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764770.087390:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978356352 00000100:00100000:1.0:1775764770.087406:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978356352 00000100:00100000:1.0:1775764770.087464:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24090 00000100:00100000:1.0:1775764770.087474:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e7800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+5:5794:x1862022978356352:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764770.087491:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bfc00/0x36f9bdc9e421c0fb lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xed7:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x2aac6a50e5f879c expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764770.087570:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764770.087577:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e7800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+5:5794:x1862022978356352:12345-192.168.202.146@tcp:104: Request processed in 105us (733us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764770.087584:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24090 00000100:00100000:1.0:1775764770.101479:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764770.101525:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764770.101622:0:11146:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1775764770.101651:0:11146:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978356480 00000100:00100000:0.0:1775764770.101660:0:11146:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978356480 00000100:00100000:0.0:1775764770.101711:0:11146:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24091 00000100:00100000:0.0:1775764770.101720:0:11146:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970e09450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:3f83546c-01a9-4156-8a97-44c446245d3e+5:5795:x1862022978356480:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764770.101776:0:11146:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c109 lrc: 2/0,0 mode: CW/CW res: [0x200000401:0xed6:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x2aac6a50e5f87b8 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.102047:0:11146:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764770.102054:0:11146:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970e09450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:3f83546c-01a9-4156-8a97-44c446245d3e+5:5795:x1862022978356480:12345-192.168.202.146@tcp:104: Request processed in 335us (583us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764770.102062:0:11146:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24091 00010000:00010000:0.0:1775764770.106568:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1775764770.106582:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d9400/0x36f9bdc9e421c110 lrc: 2/0,0 mode: CW/CW res: [0x200000401:0xed7:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x2aac6a50e5f87c6 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.106594:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f994f3d9400 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:0.0:1775764770.106599:0:5571:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d9400/0x36f9bdc9e421c110 lrc: 2/0,0 mode: CW/CW res: [0x200000401:0xed7:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x2aac6a50e5f87c6 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.106608:0:5571:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d9400/0x36f9bdc9e421c110 lrc: 2/0,0 mode: CW/CW res: [0x200000401:0xed7:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x2aac6a50e5f87c6 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.106635:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d9400/0x36f9bdc9e421c110 lrc: 3/0,0 mode: CW/CW res: [0x200000401:0xed7:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x408400000000 nid: local remote: 0x2aac6a50e5f87c6 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.106707:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d9400/0x36f9bdc9e421c110 lrc: 2/0,0 mode: --/CW res: [0x200000401:0xed7:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x2aac6a50e5f87c6 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.106714:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764770.106792:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d9400/0x36f9bdc9e421c110 lrc: 1/0,0 mode: --/CW res: [0x200000401:0xed7:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x2aac6a50e5f87c6 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.106800:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d9400/0x36f9bdc9e421c110 lrc: 0/0,0 mode: --/CW res: [0x200000401:0xed7:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x2aac6a50e5f87c6 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.106817:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764770.106818:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764770.106820:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764770.107095:0:2380:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1775764770.107116:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966900736:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764770.117609:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966900736:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764770.118784:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970c03100 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966900224:192.168.202.146@tcp:101:dbench.0 00010000:00010000:1.0:1775764770.118808:0:19723:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c117 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xed6:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f87e2 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.118821:0:19723:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c117 lrc: 4/1,0 mode: --/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f87e2 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.118834:0:19723:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c117 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f87e2 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.118845:0:19723:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c117 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f87e2 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.118858:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c117 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f87e2 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.118868:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c117 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f87e2 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.118876:0:19723:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c117 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f87e2 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764770.118886:0:19723:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970c03100 x1862022966900224/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/928 e 0 to 0 dl 1775764786 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'dbench.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764770.118897:0:19723:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c117 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f87e2 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1775764770.118908:0:19723:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c117 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f87e2 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.118920:0:19723:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/3798 (0 0) 00000080:00200000:1.0:1775764770.118927:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0xed6:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1775764770.119032:0:19723:0:(file.c:6766:ll_layout_conf()) [0x200000401:0xed6:0x0]: layout version change: 4294967294 -> 3 00000080:00010000:1.0:1775764770.119042:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xed6:0x0](ffff8f9952ee9148) for lock 0x36f9bdc9e421c117 00000080:00010000:1.0:1775764770.119048:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xed6:0x0](ffff8f9952ee9148) 00000080:00010000:1.0:1775764770.119051:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c117 from it ffffafbac1c7bcd8 00010000:00010000:1.0:1775764770.119055:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c117 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f87e2 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.119063:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c117 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f87e2 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764770.119112:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xed6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764770.119345:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xed6:0x0] 00000008:00010000:1.0:1775764770.119373:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764770.119388:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99705ee0b0, osc lock ffff8f995875c5f0, flags 0x400001002 00010000:00010000:1.0:1775764770.119696:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000aa9eb34d/0x36f9bdc9e421c0f4 lrc: 2/1,0 mode: PR/PR res: [0x280000403:0x65f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5f8780 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764770.119711:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000aa9eb34d/0x36f9bdc9e421c0f4 lrc: 2/1,0 mode: PR/PR res: [0x280000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5f8780 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:1.0:1775764770.119722:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764770.119726:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000aa9eb34d/0x36f9bdc9e421c0f4 lrc: 5/1,0 mode: PR/PR res: [0x280000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5f8780 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764770.119735:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000aa9eb34d/0x36f9bdc9e421c0f4 lrc: 4/0,0 mode: PR/PR res: [0x280000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5f8780 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:1.0:1775764770.119795:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764770.119804:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0xed6:0x0] updating i_size 0 i_blocks 1 00000020:00010000:1.0:1775764770.119808:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705f9240 (ffff8f99705b8098/0) at cl_lock_release():245 00000020:00010000:1.0:1775764770.119811:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705f9240 (ffff8f99705b8098/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764770.119814:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705ee0b0 (ffff8f99705b87e8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764770.119818:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705f9240 (ffff8f99705b8098/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764770.119820:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705ee0b0 (ffff8f99705b8098/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764770.119840:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xed6:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1775764770.119883:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file __50172.DB:[0x200000401:0xed6:0x0](ffff8f9952ee9148), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764770.119889:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xed6:0x0](ffff8f9952ee9148),name=__50172.DB 00000002:00010000:1.0:1775764770.119902:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xed6:0x0]) in obj [0x200000401:0xed6:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764770.119911:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c117 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f87e2 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.119920:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c117 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f87e2 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764770.119926:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0xed6:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764770.119930:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xed6:0x0](ffff8f9952ee9148) 00000080:00010000:1.0:1775764770.119933:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xed6:0x0](ffff8f9952ee9148) for lock 0x36f9bdc9e421c117 00000080:00010000:1.0:1775764770.119937:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c117 from it ffffafbac1c7bcd8 00010000:00010000:1.0:1775764770.119940:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c117 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f87e2 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.119947:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c117 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f87e2 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764770.119979:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xed6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764770.119983:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xed6:0x0] 00000008:00010000:1.0:1775764770.119994:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764770.119996:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99705ee0b0, osc lock ffff8f995875c5f0, flags 0x400001002 00010000:00010000:1.0:1775764770.120003:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000aa9eb34d/0x36f9bdc9e421c0f4 lrc: 2/1,0 mode: PR/PR res: [0x280000403:0x65f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5f8780 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764770.120011:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000aa9eb34d/0x36f9bdc9e421c0f4 lrc: 2/1,0 mode: PR/PR res: [0x280000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5f8780 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:1.0:1775764770.120023:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764770.120025:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000aa9eb34d/0x36f9bdc9e421c0f4 lrc: 5/1,0 mode: PR/PR res: [0x280000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5f8780 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764770.120032:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000aa9eb34d/0x36f9bdc9e421c0f4 lrc: 4/0,0 mode: PR/PR res: [0x280000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5f8780 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1775764770.120040:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764770.120044:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0xed6:0x0] updating i_size 0 i_blocks 1 00000020:00010000:1.0:1775764770.120046:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705f9240 (ffff8f99705b8098/0) at cl_lock_release():245 00000020:00010000:1.0:1775764770.120048:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705f9240 (ffff8f99705b8098/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764770.120051:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705ee0b0 (ffff8f99705b87e8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764770.120053:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705f9240 (ffff8f99705b8098/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764770.120055:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705ee0b0 (ffff8f99705b8098/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764770.120066:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xed6:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1775764770.120089:0:19723:0:(file.c:2612:do_file_write_iter()) START file __50172.DB:[0x200000401:0xed6:0x0], ppos: 2047, count: 1 00000080:00200000:1.0:1775764770.120104:0:19723:0:(rw26.c:663:ll_write_begin()) Writing 0 of 2047 to 1 bytes 00000080:00200000:1.0:1775764770.120109:0:19723:0:(file.c:2588:ll_do_tiny_write()) result: 0, original count 1 00000080:00200000:1.0:1775764770.121474:0:19723:0:(file.c:2058:ll_file_io_generic()) __50172.DB: write ppos: 2047, bytes: 1 00000080:00200000:1.0:1775764770.121478:0:19723:0:(file.c:1815:ll_io_set_mirror()) __50172.DB: desiginated mirror: 0 00000020:00200000:1.0:1775764770.121483:0:19723:0:(cl_io.c:214:cl_io_rw_init()) header@00000000f4aafcd9[0x110, 1, [0x200000401:0xed6:0x0] hash] 00000020:00200000:1.0:1775764770.121485:0:19723:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [2047, 2048) 0 0 00000080:00200000:1.0:1775764770.121487:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xed6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764770.121493:0:19723:0:(file.c:2136:ll_file_io_generic()) Range lock [0, 0] 00020000:00200000:1.0:1775764770.121496:0:19723:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 2047 chunk: [2047, 2048) 2048, 1 00020000:00200000:1.0:1775764770.121498:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764770.121516:0:19723:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:1.0:1775764770.121522:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764770.121525:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99705ee0b0, osc lock ffff8f995875c5f0, flags 0x0 00010000:00010000:1.0:1775764770.121532:0:19723:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f9951ed3c00 type 11 mode 2 res 10737419267/1631 (0 4095) 00010000:00010000:1.0:1775764770.121553:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b03c608/0x36f9bdc9e421c12c lrc: 3/0,1 mode: --/PW res: [0x280000403:0x65f:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.121562:0:19723:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b03c608/0x36f9bdc9e421c12c lrc: 3/0,1 mode: --/PW res: [0x280000403:0x65f:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.121569:0:19723:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b03c608/0x36f9bdc9e421c12c lrc: 3/0,1 mode: --/PW res: [0x280000403:0x65f:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764770.121581:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970c03100 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966900992:192.168.202.146@tcp:101:dbench.0 00000100:00100000:1.0:1775764770.121607:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f997130de40 going to sleep for 11 seconds 00000100:00100000:0.0:1775764770.125133:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764770.125169:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764770.125193:0:11131:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978356736 00000100:00100000:0.0:1775764770.125202:0:11131:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978356736 00000100:00100000:0.0:1775764770.125240:0:11131:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978356864 00000100:00100000:0.0:1775764770.125243:0:11131:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978356864 00000100:00100000:0.0:1775764770.125263:0:11131:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24092 00000100:00100000:0.0:1775764770.125271:0:11131:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e6680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:12188:x1862022978356736:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764770.125287:0:11131:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000df7db25e/0x36f9bdc9e421c102 lrc: 2/0,0 mode: PR/PR res: [0x240000403:0x660:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800420000000000 nid: local remote: 0x2aac6a50e5f87aa expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764770.125361:0:11131:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764770.125368:0:11131:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e6680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:12188:x1862022978356736:12345-192.168.202.146@tcp:104: Request processed in 98us (254us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764770.125375:0:11131:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24092 00000100:00100000:0.0:1775764770.125389:0:11131:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24093 00000100:00100000:0.0:1775764770.126269:0:11131:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e6a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:12184:x1862022978356864:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764770.126280:0:11131:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000aa9eb34d/0x36f9bdc9e421c0f4 lrc: 2/0,0 mode: PR/PR res: [0x280000403:0x65f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800420000000000 nid: local remote: 0x2aac6a50e5f8780 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764770.126354:0:11131:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764770.126360:0:11131:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e6a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:12184:x1862022978356864:12345-192.168.202.146@tcp:104: Request processed in 966us (1190us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764770.126367:0:11131:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24093 00000100:00100000:1.0:1775764770.135010:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764770.135098:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764770.135687:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978356992 00000100:00100000:0.0:1775764770.135694:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978356992 00000100:00100000:0.0:1775764770.135736:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978357120 00000100:00100000:0.0:1775764770.135758:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978357120 00000100:00100000:0.0:1775764770.135779:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24094 00000100:00100000:0.0:1775764770.135787:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970c00a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:5780:x1862022978356992:12345-192.168.202.146@tcp:105: 00010000:00010000:0.0:1775764770.135797:0:5561:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4034b9e/0x36f9bdc9e421c125 lrc: 5/0,1 mode: --/PW res: [0x240000403:0x660:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x2aac6a50e5f87f7 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.135807:0:5561:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4034b9e/0x36f9bdc9e421c125 lrc: 5/0,1 mode: --/PW res: [0x240000403:0x660:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x2aac6a50e5f87f7 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.135815:0:5561:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4034b9e/0x36f9bdc9e421c125 lrc: 5/0,1 mode: --/PW res: [0x240000403:0x660:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x2aac6a50e5f87f7 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.135830:0:5561:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4034b9e/0x36f9bdc9e421c125 lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x660:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x2aac6a50e5f87f7 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.135838:0:5561:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4034b9e/0x36f9bdc9e421c125 lrc: 6/0,1 mode: PW/PW res: [0x240000403:0x660:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x2aac6a50e5f87f7 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.135847:0:5561:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4034b9e/0x36f9bdc9e421c125 lrc: 6/0,1 mode: PW/PW res: [0x240000403:0x660:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x80000020000 nid: local remote: 0x2aac6a50e5f87f7 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.135872:0:5561:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8f9941480c00) 00000100:00100000:0.0:1775764770.135922:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970c00a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:5780:x1862022978356992:12345-192.168.202.146@tcp:105: Request processed in 135us (922us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764770.135929:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24094 00000100:00100000:0.0:1775764770.135946:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24095 00000100:00100000:0.0:1775764770.135965:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970c00380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:5779:x1862022978357120:12345-192.168.202.146@tcp:105: 00010000:00010000:0.0:1775764770.135973:0:5561:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b03c608/0x36f9bdc9e421c12c lrc: 4/0,1 mode: --/PW res: [0x280000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f87fe expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.135980:0:5561:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b03c608/0x36f9bdc9e421c12c lrc: 4/0,1 mode: --/PW res: [0x280000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f87fe expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.135987:0:5561:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b03c608/0x36f9bdc9e421c12c lrc: 4/0,1 mode: --/PW res: [0x280000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f87fe expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.135994:0:5561:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b03c608/0x36f9bdc9e421c12c lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000000000 nid: local remote: 0x2aac6a50e5f87fe expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.136001:0:5561:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b03c608/0x36f9bdc9e421c12c lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000000000 nid: local remote: 0x2aac6a50e5f87fe expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.136008:0:5561:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b03c608/0x36f9bdc9e421c12c lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x80000000000 nid: local remote: 0x2aac6a50e5f87fe expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.136017:0:5561:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8f99668bfe00) 00000100:00100000:0.0:1775764770.136034:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970c00380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:5779:x1862022978357120:12345-192.168.202.146@tcp:105: Request processed in 70us (936us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764770.136039:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24095 00000100:00100000:0.0:1775764770.149960:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703df800 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966901888:192.168.202.146@tcp:101:dbench.0 00010000:00010000:0.0:1775764770.149985:0:19723:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c133 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f881a expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.150000:0:19723:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c133 lrc: 4/1,0 mode: --/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f881a expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.150009:0:19723:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c133 lrc: 4/1,0 mode: --/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f881a expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.150020:0:19723:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c133 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f881a expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.150031:0:19723:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c133 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f881a expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.150048:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c133 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f881a expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.150057:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c133 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f881a expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.150064:0:19723:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c133 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f881a expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764770.150074:0:19723:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f99703df800 x1862022966901888/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/928 e 0 to 0 dl 1775764786 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'dbench.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1775764770.150085:0:19723:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c133 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f881a expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:0.0:1775764770.150098:0:19723:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c133 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f881a expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.150110:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c117 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f87e2 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764770.150219:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c117 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f87e2 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764770.150228:0:19723:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c133 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f881a expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.150234:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c133 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x2aac6a50e5f881a expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.150241:0:19723:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c133 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x2aac6a50e5f881a expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.150270:0:19723:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000080:00200000:0.0:1775764770.150278:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2bb:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1775764770.150381:0:19723:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0xed6:0x0](ffff8f9952ee9148): rc = 0 00000080:00200000:0.0:1775764770.150414:0:19723:0:(file.c:6766:ll_layout_conf()) [0x200000401:0xed6:0x0]: layout version change: 3 -> 3 00000080:00010000:0.0:1775764770.150425:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xed6:0x0](ffff8f9952ee9148) for lock 0x36f9bdc9e421c117 00000080:00010000:0.0:1775764770.150436:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xed6:0x0](ffff8f9952ee9148) 00000080:00010000:0.0:1775764770.150440:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c117 from it ffffafbac1c7bb20 00010000:00010000:0.0:1775764770.150443:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c117 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f87e2 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764770.150452:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c117 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f87e2 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200008:0.0:1775764770.150504:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file __50172.DB:[0x200000401:0xed6:0x0](ffff8f9952ee9148), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764770.150511:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xed6:0x0](ffff8f9952ee9148),name=__50172.DB 00000002:00010000:0.0:1775764770.150530:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xed6:0x0]) in obj [0x200000401:0x2bb:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764770.150538:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c117 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f87e2 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764770.150547:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c117 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f87e2 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:0.0:1775764770.150553:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2bb:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764770.150558:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xed6:0x0](ffff8f9952ee9148) 00000080:00010000:0.0:1775764770.150561:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xed6:0x0](ffff8f9952ee9148) for lock 0x36f9bdc9e421c117 00000080:00010000:0.0:1775764770.150564:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c117 from it ffffafbac1c7bcb8 00010000:00010000:0.0:1775764770.150566:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c117 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f87e2 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764770.150573:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c117 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f87e2 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00200000:0.0:1775764770.150608:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xed6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1775764770.150618:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xed6:0x0] 00000008:00010000:0.0:1775764770.150660:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1775764770.150664:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0dfcb0, osc lock ffff8f996686f430, flags 0x400001002 00010000:00010000:0.0:1775764770.150680:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b03c608/0x36f9bdc9e421c12c lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x65f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f87fe expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764770.150691:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b03c608/0x36f9bdc9e421c12c lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f87fe expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:0.0:1775764770.150702:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1775764770.150705:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b03c608/0x36f9bdc9e421c12c lrc: 6/0,1 mode: PW/PW res: [0x280000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5f87fe expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764770.150713:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b03c608/0x36f9bdc9e421c12c lrc: 5/0,0 mode: PW/PW res: [0x280000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f87fe expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:0.0:1775764770.150722:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1775764770.150732:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0xed6:0x0] updating i_size 2048 i_blocks 1 00000020:00010000:0.0:1775764770.150762:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970b8ab58 (ffff8f9966dc2638/0) at cl_lock_release():245 00000020:00010000:0.0:1775764770.150765:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970b8ab58 (ffff8f9966dc2638/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.150768:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0dfcb0 (ffff8f9966dc2bd8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.150772:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970b8ab58 (ffff8f9966dc2638/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764770.150775:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0dfcb0 (ffff8f9966dc2638/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764770.150791:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xed6:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764770.150837:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.150842:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.150846:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:0.0:1775764770.150851:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.150854:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:0.0:1775764770.150857:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.150862:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:0.0:1775764770.150865:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.150868:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:0.0:1775764770.150870:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.150873:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=__50172.DB, flags=257 00000080:00200000:0.0:1775764770.150877:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xed6:0x0](ffff8f9952ee9148), inode mode 8180 mask 46 00000080:00200008:0.0:1775764770.150886:0:19723:0:(file.c:1057:ll_file_open()) START file __50172.DB:[0x200000401:0xed6:0x0](ffff8f9952ee9148), flags 100002 00000002:00010000:0.0:1775764770.150900:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xed6:0x0]) in obj [0x200000401:0x2bb:0x0], intent: open flags 04240100003 00010000:00010000:0.0:1775764770.150954:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c141 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.150962:0:19723:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c141 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1775764770.150971:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1775764770.150979:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e7800 x1862022966902272/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/135824 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1775764770.150994:0:19723:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c141 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764770.151013:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966902272:192.168.202.146@tcp:101:dbench.0 00010000:00010000:1.0:1775764770.151927:0:5572:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764770.151937:0:5572:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c133 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f881a expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.151947:0:5572:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9941480a00 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:1.0:1775764770.151961:0:5572:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c133 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f881a expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.151970:0:5572:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c133 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f881a expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.151978:0:5572:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c133 lrc: 3/0,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f881a expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.152003:0:5572:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c117 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f87e2 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1775764770.152037:0:5572:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c133 lrc: 2/0,0 mode: --/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f881a expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.152043:0:5572:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764770.152070:0:5572:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c133 lrc: 1/0,0 mode: --/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f881a expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.152077:0:5572:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c133 lrc: 0/0,0 mode: --/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f881a expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.152092:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764770.152094:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764770.152096:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764770.152113:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970c03480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966902400:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1775764770.153725:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764770.153731:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941481c00/0x36f9bdc9e421c13a lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xed7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f8836 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.153757:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9941481c00 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:1.0:1775764770.153761:0:19801:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941481c00/0x36f9bdc9e421c13a lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xed7:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f8836 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.153768:0:19801:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941481c00/0x36f9bdc9e421c13a lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xed7:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f8836 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.153780:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941481c00/0x36f9bdc9e421c13a lrc: 3/0,0 mode: PR/PR res: [0x200000401:0xed7:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f8836 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.153795:0:19801:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480400/0x36f9bdc9e421c11e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xed7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f87f0 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764770.153819:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941481c00/0x36f9bdc9e421c13a lrc: 2/0,0 mode: --/PR res: [0x200000401:0xed7:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8836 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.153825:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764770.153842:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941481c00/0x36f9bdc9e421c13a lrc: 1/0,0 mode: --/PR res: [0x200000401:0xed7:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8836 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.153849:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941481c00/0x36f9bdc9e421c13a lrc: 0/0,0 mode: --/PR res: [0x200000401:0xed7:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8836 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.153864:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764770.153866:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764770.153868:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764770.153878:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970c03100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966902784:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764770.158521:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16fb40 going to sleep for 11 seconds 00000100:00100000:0.0:1775764770.159661:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764770.159699:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978357248 00000100:00100000:0.0:1775764770.159710:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978357248 00000100:00100000:0.0:1775764770.159785:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24096 00000100:00100000:0.0:1775764770.159795:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e7100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+5:9373:x1862022978357248:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764770.159806:0:5561:0:(ldlm_lockd.c:2448:ldlm_callback_handler()) callback on lock 0x36f9bdc9e421c13a - lock disappeared 00010000:00010000:0.0:1775764770.159848:0:5561:0:(ldlm_lockd.c:2345:ldlm_callback_errmsg()) @@@ Operate with invalid parameter, NID=12345-192.168.202.146@tcp lock=0x36f9bdc9e421c13a: rc = 0 req@ffff8f994f2e7100 x1862022978357248/t0(0) o104->3f83546c-01a9-4156-8a97-44c446245d3e@192.168.202.146@tcp:251/0 lens 328/224 e 0 to 0 dl 1775764781 ref 1 fl Interpret:/0/0 rc -22/-22 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764770.159860:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e7100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+5:9373:x1862022978357248:12345-192.168.202.146@tcp:104: Request processed in 67us (208us total) trans 0 rc -22/-22 00000100:00100000:0.0:1775764770.159866:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24096 00000100:00100000:0.0:1775764770.159954:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764770.160002:0:5562:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1775764770.160015:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978357376 00000100:00100000:0.0:1775764770.160018:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978357376 00000100:00100000:0.0:1775764770.160039:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24097 00000100:00100000:0.0:1775764770.160045:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9966931850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+5:9373:x1862022978357376:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764770.160058:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480400/0x36f9bdc9e421c11e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xed7:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x2aac6a50e5f87f0 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:0.0:1775764770.160097:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764770.160103:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9966931850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+5:9373:x1862022978357376:12345-192.168.202.146@tcp:104: Request processed in 58us (155us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764770.160109:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24097 00000100:00100000:1.0:1775764770.160382:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970c03480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966902400:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764770.169926:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764770.169966:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978357504 00000100:00100000:0.0:1775764770.169976:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978357504 00000100:00100000:0.0:1775764770.170028:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24098 00000100:00100000:0.0:1775764770.170040:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e6680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+5:5794:x1862022978357504:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764770.170059:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c117 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x2aac6a50e5f87e2 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 1 00000100:00100000:1.0:1775764770.170341:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970c03100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966902784:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1775764770.171439:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764770.171452:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e6680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+5:5794:x1862022978357504:12345-192.168.202.146@tcp:104: Request processed in 1412us (1533us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764770.171462:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24098 00010000:00010000:0.0:1775764770.171497:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1775764770.171504:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c117 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x2aac6a50e5f87e2 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 1 00010000:00010000:0.0:1775764770.171514:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99668be600 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:0.0:1775764770.171519:0:5571:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c117 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x2aac6a50e5f87e2 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 1 00010000:00010000:0.0:1775764770.171528:0:5571:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c117 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x2aac6a50e5f87e2 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 1 00010000:00010000:0.0:1775764770.171542:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c117 lrc: 3/0,0 mode: PR/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x2aac6a50e5f87e2 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 1 00000080:00200000:0.0:1775764770.171590:0:5571:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0xed6:0x0]: losing layout lock 00010000:00010000:0.0:1775764770.171709:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c117 lrc: 2/0,0 mode: --/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x2aac6a50e5f87e2 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 1 00010000:00010000:0.0:1775764770.171716:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764770.171753:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c117 lrc: 1/0,0 mode: --/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x2aac6a50e5f87e2 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 1 00010000:00010000:0.0:1775764770.171761:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c117 lrc: 0/0,0 mode: --/PR res: [0x200000401:0xed6:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x2aac6a50e5f87e2 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 1 00010000:00010000:0.0:1775764770.171778:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764770.171780:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764770.171782:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764770.171995:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1775764770.172010:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966903040:192.168.202.146@tcp:103:ldlm_bl.0 00000080:00200008:0.0:1775764770.176850:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file __52E2B.DB:[0x200000401:0xed7:0x0](ffff8f9952eb4b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764770.176863:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xed7:0x0](ffff8f9952eb4b08),name=__52E2B.DB 00000002:00010000:0.0:1775764770.176880:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xed7:0x0]) in obj [0x200000401:0xed7:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764770.176892:0:19724:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/3799 (0 0) 00010000:00010000:0.0:1775764770.176941:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5800/0x36f9bdc9e421c14f lrc: 3/1,0 mode: --/CR res: [0x200000401:0xed7:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.176961:0:19724:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5800/0x36f9bdc9e421c14f lrc: 3/1,0 mode: --/CR res: [0x200000401:0xed7:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.176971:0:19724:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5800/0x36f9bdc9e421c14f lrc: 3/1,0 mode: --/CR res: [0x200000401:0xed7:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764770.176999:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7100 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966903168:192.168.202.146@tcp:101:dbench.0 00000100:00100000:0.0:1775764770.177091:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f000 going to sleep for 11 seconds 00000100:00100000:1.0:1775764770.182677:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764770.182710:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978357632 00000100:00100000:1.0:1775764770.182719:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978357632 00000100:00100000:1.0:1775764770.182805:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24099 00000100:00100000:1.0:1775764770.182816:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970c03100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+5:5795:x1862022978357632:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764770.182834:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971078600/0x36f9bdc9e421c148 lrc: 2/0,0 mode: CW/CW res: [0x200000401:0xed7:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x2aac6a50e5f8844 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.182912:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764770.182919:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970c03100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+5:5795:x1862022978357632:12345-192.168.202.146@tcp:104: Request processed in 106us (251us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764770.182927:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24099 00000100:00100000:0.0:1775764770.183431:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966903040:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764770.191833:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764770.191867:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978357760 00000100:00100000:0.0:1775764770.191877:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978357760 00000100:00100000:0.0:1775764770.191921:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24100 00000100:00100000:0.0:1775764770.191930:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e6a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+5:5796:x1862022978357760:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764770.191954:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c141 lrc: 2/0,0 mode: CW/CW res: [0x200000401:0xed6:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x2aac6a50e5f8852 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.192013:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764770.192020:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e6a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+5:5796:x1862022978357760:12345-192.168.202.146@tcp:104: Request processed in 91us (198us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764770.192027:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24100 00010000:00010000:1.0:1775764770.206660:0:5572:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764770.206667:0:5572:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971078600/0x36f9bdc9e421c16b lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x267:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f8891 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.206675:0:5572:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9971078600 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:1.0:1775764770.206679:0:5572:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971078600/0x36f9bdc9e421c16b lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x267:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f8891 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.206687:0:5572:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971078600/0x36f9bdc9e421c16b lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x267:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f8891 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.206695:0:5572:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971078600/0x36f9bdc9e421c16b lrc: 3/0,0 mode: CR/CR res: [0x200000401:0x267:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f8891 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.206711:0:5572:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d8400/0x36f9bdc9e421c15d lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x267:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8883 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764770.206767:0:5572:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971078600/0x36f9bdc9e421c16b lrc: 2/0,0 mode: --/CR res: [0x200000401:0x267:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8891 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.206774:0:5572:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764770.206804:0:5572:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971078600/0x36f9bdc9e421c16b lrc: 1/0,0 mode: --/CR res: [0x200000401:0x267:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8891 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.206811:0:5572:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971078600/0x36f9bdc9e421c16b lrc: 0/0,0 mode: --/CR res: [0x200000401:0x267:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8891 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.206825:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764770.206827:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764770.206828:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764770.206835:0:2380:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1775764770.206850:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970c03480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966904064:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764770.221935:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970c03480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966904064:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1775764770.259563:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764770.259574:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971079c00/0x36f9bdc9e421c180 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x265:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f88a6 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.259584:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9971079c00 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:1.0:1775764770.259589:0:19801:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971079c00/0x36f9bdc9e421c180 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x265:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f88a6 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.259597:0:19801:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971079c00/0x36f9bdc9e421c180 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x265:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f88a6 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.259622:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971079c00/0x36f9bdc9e421c180 lrc: 3/0,0 mode: CR/CR res: [0x200000401:0x265:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f88a6 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.259652:0:19801:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d8e00/0x36f9bdc9e421c172 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x265:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8898 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764770.259691:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971079c00/0x36f9bdc9e421c180 lrc: 2/0,0 mode: --/CR res: [0x200000401:0x265:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f88a6 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.259697:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764770.259727:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971079c00/0x36f9bdc9e421c180 lrc: 1/0,0 mode: --/CR res: [0x200000401:0x265:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f88a6 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.259734:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971079c00/0x36f9bdc9e421c180 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x265:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f88a6 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.259763:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764770.259765:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764770.259767:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764770.259785:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970c03480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966904960:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764770.264992:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970c03480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966904960:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1775764770.271545:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 4/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764770.271559:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 8/3,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764770.271572:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 8/3,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764770.271580:0:19724:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 7/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764770.271593:0:19724:0:(vvp_io.c:178:vvp_prep_size()) [0x200000401:0x266:0x0] updating i_size 2097152 00000020:00010000:1.0:1775764770.275970:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99587d54e8 (ffff8f9970314248/0) at cl_lock_release():245 00000020:00010000:1.0:1775764770.275975:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99587d54e8 (ffff8f9970314248/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764770.275979:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d5030 (ffff8f99703143f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764770.275988:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 6/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764770.276002:0:19724:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 5/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00010000:1.0:1775764770.276010:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99587d54e8 (ffff8f9970314248/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764770.276013:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d5030 (ffff8f9970314248/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764770.276052:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x266:0x0] ignore/verify layout 0/0, layout version 4 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764770.276170:0:19724:0:(rw.c:1928:ll_readpage()) fast read pgno: 1 00000080:00200000:1.0:1775764770.276179:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x266:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764770.276191:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764770.276214:0:19724:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [1, 1] 00000008:00010000:1.0:1775764770.276228:0:19724:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764770.276231:0:19724:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99705eea30, osc lock ffff8f995875da20, flags 0x0 00010000:00010000:1.0:1775764770.276251:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 4/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 4 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764770.276266:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (4096 8191) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 4/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764770.276279:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 8/3,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764770.276288:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 8/3,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764770.276296:0:19724:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 7/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764770.276308:0:19724:0:(vvp_io.c:178:vvp_prep_size()) [0x200000401:0x266:0x0] updating i_size 2097152 00000008:00010000:1.0:1775764770.276353:0:19724:0:(osc_cache.c:2998:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8f9970a083c0 00000008:00010000:1.0:1775764770.276403:0:21:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8f9970a083c0 00010000:00010000:1.0:1775764770.276426:0:21:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (4096 8191) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 6/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000008:00100000:1.0:1775764770.276467:0:21:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8f9970c03480 - object 0x240000403:50 offset 4096<>8192 00000100:00100000:1.0:1775764770.276520:0:2380:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1775764770.276534:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970c03480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966905216:192.168.202.146@tcp:3:dbench.0 00000008:00010000:0.0:1775764770.283174:0:2380:0:(osc_cache.c:901:osc_extent_finish()) ### no dlm lock: extent: ffff8f9970a083c0 00000020:00200000:0.0:1775764770.283188:0:2380:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1775764770.283206:0:2380:0:(osc_cache.c:496:__osc_extent_remove()) ### no dlm lock: extent: ffff8f9970a083c0 00000100:00100000:0.0:1775764770.283230:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970c03480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966905216:192.168.202.146@tcp:3:dbench.0 00000020:00010000:1.0:1775764770.283453:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705baa88 (ffff8f99705b8bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764770.283457:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705baa88 (ffff8f99705b8bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764770.283461:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705eea30 (ffff8f99705b8f38/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764770.283468:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 6/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764770.283482:0:19724:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 5/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00010000:1.0:1775764770.283490:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705baa88 (ffff8f99705b8bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764770.283493:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705eea30 (ffff8f99705b8bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764770.283520:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x266:0x0] ignore/verify layout 0/0, layout version 4 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764770.283603:0:19724:0:(rw.c:1928:ll_readpage()) fast read pgno: 2 00000080:00200000:1.0:1775764770.283620:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x266:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764770.283637:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764770.283658:0:19724:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [2, 2] 00000008:00010000:1.0:1775764770.283666:0:19724:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764770.283670:0:19724:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99705eea30, osc lock ffff8f995875da20, flags 0x0 00010000:00010000:1.0:1775764770.283685:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 4/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 4 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764770.283696:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (8192 12287) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 4/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764770.283706:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 8/3,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764770.283714:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 8/3,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764770.283721:0:19724:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 7/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764770.283768:0:19724:0:(vvp_io.c:178:vvp_prep_size()) [0x200000401:0x266:0x0] updating i_size 2097152 00000008:00010000:1.0:1775764770.283803:0:19724:0:(osc_cache.c:2998:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8f9970a08840 00000008:00010000:1.0:1775764770.283838:0:21:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8f9970a08840 00010000:00010000:1.0:1775764770.283858:0:21:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (8192 12287) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 6/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000008:00100000:1.0:1775764770.283893:0:21:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8f9970c03b80 - object 0x240000403:50 offset 8192<>12288 00000100:00100000:0.0:1775764770.284560:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1775764770.284578:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970c03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966905344:192.168.202.146@tcp:3:dbench.0 00000008:00010000:0.0:1775764770.288900:0:2379:0:(osc_cache.c:901:osc_extent_finish()) ### no dlm lock: extent: ffff8f9970a08840 00000020:00200000:0.0:1775764770.288915:0:2379:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1775764770.289044:0:2379:0:(osc_cache.c:496:__osc_extent_remove()) ### no dlm lock: extent: ffff8f9970a08840 00000100:00100000:0.0:1775764770.289069:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970c03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966905344:192.168.202.146@tcp:3:dbench.0 00000020:00010000:1.0:1775764770.289300:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705baa88 (ffff8f99705b8bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764770.289306:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705baa88 (ffff8f99705b8bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764770.289310:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705eea30 (ffff8f99705b8f38/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764770.289318:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 6/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764770.289330:0:19724:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 5/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00010000:1.0:1775764770.289339:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705baa88 (ffff8f99705b8bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764770.289342:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705eea30 (ffff8f99705b8bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764770.289368:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x266:0x0] ignore/verify layout 0/0, layout version 4 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764770.289778:0:19724:0:(rw.c:1928:ll_readpage()) fast read pgno: 11 00000080:00200000:1.0:1775764770.289786:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x266:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764770.289799:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764770.289822:0:19724:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [11, 11] 00000008:00010000:1.0:1775764770.289833:0:19724:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764770.289838:0:19724:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99705eea30, osc lock ffff8f995875da20, flags 0x0 00010000:00010000:1.0:1775764770.289855:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 2/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 4 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764770.289871:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (45056 49151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 2/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764770.289882:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 6/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764770.289890:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 6/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764770.289897:0:19724:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 5/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764770.289910:0:19724:0:(vvp_io.c:178:vvp_prep_size()) [0x200000401:0x266:0x0] updating i_size 2097152 00000008:00010000:1.0:1775764770.289944:0:19724:0:(osc_cache.c:2998:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8f9970a08b40 00000008:00010000:1.0:1775764770.290321:0:21:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8f9970a08b40 00010000:00010000:1.0:1775764770.290344:0:21:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (45056 49151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 6/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000008:00100000:1.0:1775764770.290383:0:21:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8f9970c02d80 - object 0x240000403:50 offset 45056<>49152 00000100:00100000:0.0:1775764770.290541:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970c02d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966905472:192.168.202.146@tcp:3:dbench.0 00000008:00010000:0.0:1775764770.295004:0:2379:0:(osc_cache.c:901:osc_extent_finish()) ### no dlm lock: extent: ffff8f9970a08b40 00000020:00200000:0.0:1775764770.295018:0:2379:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1775764770.295040:0:2379:0:(osc_cache.c:496:__osc_extent_remove()) ### no dlm lock: extent: ffff8f9970a08b40 00000100:00100000:0.0:1775764770.295062:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970c02d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966905472:192.168.202.146@tcp:3:dbench.0 00000020:00010000:1.0:1775764770.301691:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970942998 (ffff8f994ef58908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764770.301696:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970942998 (ffff8f994ef58908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764770.301700:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994166feb0 (ffff8f994ef58758/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764770.301711:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 6/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764770.301725:0:19724:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 5/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000020:00010000:1.0:1775764770.301734:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970942998 (ffff8f994ef58908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764770.301771:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994166feb0 (ffff8f994ef58908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764770.301799:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x266:0x0] ignore/verify layout 0/0, layout version 4 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764770.301913:0:19724:0:(rw.c:1928:ll_readpage()) fast read pgno: 24 00000080:00200000:1.0:1775764770.301923:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x266:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764770.301937:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764770.301962:0:19724:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [24, 24] 00000008:00010000:1.0:1775764770.301975:0:19724:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764770.301978:0:19724:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99705eedb0, osc lock ffff8f995875c720, flags 0x0 00010000:00010000:1.0:1775764770.301995:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 4/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 4 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764770.302007:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98304 102399) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 4/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764770.302017:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 8/3,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764770.302025:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 8/3,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764770.302031:0:19724:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 7/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000080:00200000:1.0:1775764770.302043:0:19724:0:(vvp_io.c:178:vvp_prep_size()) [0x200000401:0x266:0x0] updating i_size 2097152 00000008:00010000:1.0:1775764770.302077:0:19724:0:(osc_cache.c:2998:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8f9970a08e40 00000008:00010000:1.0:1775764770.302111:0:21:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8f9970a08e40 00010000:00010000:1.0:1775764770.302144:0:21:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98304 102399) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 6/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000008:00100000:1.0:1775764770.302176:0:21:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8f9970c02d80 - object 0x240000403:50 offset 98304<>102400 00000100:00100000:0.0:1775764770.302509:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970c02d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966905728:192.168.202.146@tcp:3:dbench.0 00000008:00010000:1.0:1775764770.307433:0:2380:0:(osc_cache.c:901:osc_extent_finish()) ### no dlm lock: extent: ffff8f9970a08e40 00000020:00200000:1.0:1775764770.307449:0:2380:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1775764770.307471:0:2380:0:(osc_cache.c:496:__osc_extent_remove()) ### no dlm lock: extent: ffff8f9970a08e40 00000100:00100000:1.0:1775764770.307494:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970c02d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966905728:192.168.202.146@tcp:3:dbench.0 00000020:00010000:1.0:1775764770.313629:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970942218 (ffff8f994ef585a8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764770.313635:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970942218 (ffff8f994ef585a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764770.313640:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994166f030 (ffff8f994ef582d8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764770.313651:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 6/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764770.313667:0:19724:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 5/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000020:00010000:1.0:1775764770.313675:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970942218 (ffff8f994ef585a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764770.313678:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994166f030 (ffff8f994ef585a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764770.313710:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x266:0x0] ignore/verify layout 0/0, layout version 4 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764770.314015:0:19724:0:(rw.c:1928:ll_readpage()) fast read pgno: 148 00000080:00200000:0.0:1775764770.314025:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x266:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764770.314038:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764770.314064:0:19724:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [148, 148] 00000008:00010000:0.0:1775764770.314077:0:19724:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764770.314082:0:19724:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0dfcb0, osc lock ffff8f996686f430, flags 0x0 00010000:00010000:0.0:1775764770.314102:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 4/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 4 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:0.0:1775764770.314128:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (606208 610303) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 4/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:0.0:1775764770.314142:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 8/3,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:0.0:1775764770.314151:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 8/3,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:0.0:1775764770.314158:0:19724:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 7/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000080:00200000:0.0:1775764770.314176:0:19724:0:(vvp_io.c:178:vvp_prep_size()) [0x200000401:0x266:0x0] updating i_size 2097152 00000008:00010000:0.0:1775764770.314217:0:19724:0:(osc_cache.c:2998:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8f9982040300 00000008:00010000:0.0:1775764770.314261:0:131:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8f9982040300 00010000:00010000:0.0:1775764770.314283:0:131:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (606208 610303) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 6/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000008:00100000:0.0:1775764770.314330:0:131:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8f994f2e4380 - object 0x240000403:50 offset 606208<>610304 00000100:00100000:1.0:1775764770.314775:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966905984:192.168.202.146@tcp:3:dbench.0 00000008:00010000:1.0:1775764770.320105:0:2379:0:(osc_cache.c:901:osc_extent_finish()) ### no dlm lock: extent: ffff8f9982040300 00000020:00200000:1.0:1775764770.320217:0:2379:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1775764770.320237:0:2379:0:(osc_cache.c:496:__osc_extent_remove()) ### no dlm lock: extent: ffff8f9982040300 00000100:00100000:1.0:1775764770.320261:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966905984:192.168.202.146@tcp:3:dbench.0 00000020:00010000:0.0:1775764770.320435:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99713e93f8 (ffff8f9966dc2638/0) at cl_lock_release():245 00000020:00010000:0.0:1775764770.320441:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99713e93f8 (ffff8f9966dc2638/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.320445:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0dfcb0 (ffff8f9966dc2bd8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764770.320454:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 6/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:0.0:1775764770.320467:0:19724:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 5/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000020:00010000:0.0:1775764770.320476:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99713e93f8 (ffff8f9966dc2638/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764770.320480:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0dfcb0 (ffff8f9966dc2638/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764770.320509:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x266:0x0] ignore/verify layout 0/0, layout version 4 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:0.0:1775764770.320604:0:19724:0:(file.c:447:ll_file_release()) START file client.txt:[0x200000401:0x262:0x0](ffff8f9952db1148), flags 100000 00000080:00200000:0.0:1775764770.322225:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x266:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764770.322235:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764770.322251:0:19724:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [4, 4] 00000008:00010000:0.0:1775764770.322259:0:19724:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764770.322262:0:19724:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0dfcb0, osc lock ffff8f996686f430, flags 0x0 00010000:00010000:0.0:1775764770.322272:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 4/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 4 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:0.0:1775764770.322284:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (16384 20479) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 4/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:0.0:1775764770.322293:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 8/3,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:0.0:1775764770.322300:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 8/3,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:0.0:1775764770.322307:0:19724:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 7/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00000080:00200000:0.0:1775764770.322318:0:19724:0:(vvp_io.c:178:vvp_prep_size()) [0x200000401:0x266:0x0] updating i_size 2097152 00000020:00010000:0.0:1775764770.332402:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970942e48 (ffff8f994ef58128/0) at cl_lock_release():245 00000020:00010000:0.0:1775764770.332412:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970942e48 (ffff8f994ef58128/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.332417:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994166fcb0 (ffff8f994ef58a28/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764770.332428:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 6/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:0.0:1775764770.332441:0:19723:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 5/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00000020:00010000:0.0:1775764770.332449:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970942e48 (ffff8f994ef58128/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764770.332452:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994166fcb0 (ffff8f994ef58128/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764770.332487:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x266:0x0] ignore/verify layout 0/0, layout version 4 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764770.332581:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x266:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764770.332593:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764770.332793:0:19723:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [20, 20] 00000008:00010000:0.0:1775764770.332809:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764770.332813:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0df730, osc lock ffff8f996686eab0, flags 0x0 00010000:00010000:0.0:1775764770.332834:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 4/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 4 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:0.0:1775764770.332847:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (81920 86015) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 4/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:0.0:1775764770.332858:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 8/3,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:0.0:1775764770.332867:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 8/3,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:0.0:1775764770.332874:0:19723:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 7/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00000080:00200000:0.0:1775764770.332887:0:19723:0:(vvp_io.c:178:vvp_prep_size()) [0x200000401:0x266:0x0] updating i_size 2097152 00000020:00010000:0.0:1775764770.335098:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99709426c8 (ffff8f994ef58bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764770.335102:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99709426c8 (ffff8f994ef58bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.335106:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994166ffb0 (ffff8f994ef58b48/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764770.335123:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 6/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:0.0:1775764770.335137:0:19724:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 5/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00000020:00010000:0.0:1775764770.335146:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99709426c8 (ffff8f994ef58bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764770.335149:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994166ffb0 (ffff8f994ef58bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764770.335180:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x266:0x0] ignore/verify layout 0/0, layout version 4 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764770.335295:0:19724:0:(rw.c:1928:ll_readpage()) fast read pgno: 334 00000080:00200000:0.0:1775764770.335303:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x266:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764770.335313:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764770.335337:0:19724:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [334, 334] 00000008:00010000:0.0:1775764770.335351:0:19724:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764770.335355:0:19724:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0dfe30, osc lock ffff8f996686f430, flags 0x0 00010000:00010000:0.0:1775764770.335374:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4f14cf5/0x36f9bdc9e4211ec0 lrc: 2/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 1232896->1236991) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 0 00010000:00010000:0.0:1775764770.335388:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (1368064 1372159) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4f14cf5/0x36f9bdc9e4211ec0 lrc: 2/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1232896->1236991) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 0 00010000:00010000:0.0:1775764770.335411:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4f14cf5/0x36f9bdc9e4211ec0 lrc: 6/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1232896->1236991) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 0 00010000:00010000:0.0:1775764770.335420:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4f14cf5/0x36f9bdc9e4211ec0 lrc: 6/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1232896->1236991) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 0 00010000:00010000:0.0:1775764770.335426:0:19724:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4f14cf5/0x36f9bdc9e4211ec0 lrc: 5/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1232896->1236991) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 0 00000080:00200000:0.0:1775764770.335440:0:19724:0:(vvp_io.c:178:vvp_prep_size()) [0x200000401:0x266:0x0] updating i_size 2097152 00000008:00010000:0.0:1775764770.335487:0:19724:0:(osc_cache.c:2998:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8f9982040600 00000008:00010000:0.0:1775764770.335526:0:131:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8f9982040600 00010000:00010000:0.0:1775764770.335547:0:131:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (1368064 1372159) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4f14cf5/0x36f9bdc9e4211ec0 lrc: 4/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1232896->1236991) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 0 00000008:00100000:0.0:1775764770.335585:0:131:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8f994f2e6d80 - object 0x240000403:50 offset 1368064<>1372160 00000100:00100000:1.0:1775764770.335990:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966906368:192.168.202.146@tcp:3:dbench.0 00000020:00010000:1.0:1775764770.336108:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99713e9d58 (ffff8f9966dc2ea8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764770.336111:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99713e9d58 (ffff8f9966dc2ea8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764770.336124:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0df730 (ffff8f9966dc2518/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764770.336129:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 4/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764770.336139:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 3/0,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00000020:00010000:1.0:1775764770.336153:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99713e9d58 (ffff8f9966dc2ea8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764770.336155:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0df730 (ffff8f9966dc2ea8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764770.336177:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x266:0x0] ignore/verify layout 0/0, layout version 4 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764770.336246:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x266:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764770.336252:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764770.336267:0:19723:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [334, 334] 00000008:00010000:1.0:1775764770.336274:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764770.336276:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99705ee430, osc lock ffff8f995875d8f0, flags 0x0 00010000:00010000:1.0:1775764770.336286:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4f14cf5/0x36f9bdc9e4211ec0 lrc: 4/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 1232896->1236991) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 0 00010000:00010000:1.0:1775764770.336295:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (1368064 1372159) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4f14cf5/0x36f9bdc9e4211ec0 lrc: 4/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1232896->1236991) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 0 00010000:00010000:1.0:1775764770.336303:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4f14cf5/0x36f9bdc9e4211ec0 lrc: 8/3,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1232896->1236991) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 0 00010000:00010000:1.0:1775764770.336315:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4f14cf5/0x36f9bdc9e4211ec0 lrc: 8/3,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1232896->1236991) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 0 00010000:00010000:1.0:1775764770.336322:0:19723:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4f14cf5/0x36f9bdc9e4211ec0 lrc: 7/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1232896->1236991) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 0 00000080:00200000:1.0:1775764770.336330:0:19723:0:(vvp_io.c:178:vvp_prep_size()) [0x200000401:0x266:0x0] updating i_size 2097152 00000008:00010000:1.0:1775764770.339349:0:2380:0:(osc_cache.c:901:osc_extent_finish()) ### no dlm lock: extent: ffff8f9982040600 00000020:00200000:1.0:1775764770.339365:0:2380:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1775764770.339384:0:2380:0:(osc_cache.c:496:__osc_extent_remove()) ### no dlm lock: extent: ffff8f9982040600 00000020:00010000:0.0:1775764770.339523:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99713e9b78 (ffff8f9966dc2368/0) at cl_lock_release():245 00000020:00010000:0.0:1775764770.339529:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99713e9b78 (ffff8f9966dc2368/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.339534:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0dfe30 (ffff8f9966dc2128/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764770.339543:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4f14cf5/0x36f9bdc9e4211ec0 lrc: 6/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1232896->1236991) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 0 00010000:00010000:0.0:1775764770.339558:0:19724:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4f14cf5/0x36f9bdc9e4211ec0 lrc: 5/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1232896->1236991) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 0 00000020:00010000:0.0:1775764770.339567:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99713e9b78 (ffff8f9966dc2368/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764770.339570:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0dfe30 (ffff8f9966dc2368/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764770.340712:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x266:0x0] ignore/verify layout 0/0, layout version 4 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764770.340821:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.340825:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.340829:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=19 00000080:00200000:0.0:1775764770.340833:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.340837:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=259 00000080:00200000:0.0:1775764770.340840:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 44 00000080:00200008:0.0:1775764770.340845:0:19724:0:(file.c:1057:ll_file_open()) START file client1:[0x200000401:0x269:0x0](ffff8f9952db63c8), flags 304000 00000002:00010000:0.0:1775764770.340857:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x269:0x0]) in obj [0x200000401:0x268:0x0], intent: open flags 0100240304001 00010000:00010000:0.0:1775764770.340889:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c18e lrc: 3/1,0 mode: --/CR res: [0x200000401:0x268:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.340902:0:19724:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c18e lrc: 3/1,0 mode: --/CR res: [0x200000401:0x268:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1775764770.340909:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 2 is allocated opc 101, max 7 00000100:00100000:0.0:1775764770.340915:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e5f80 x1862022966906880/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/135824 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1775764770.340924:0:19724:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c18e lrc: 3/1,0 mode: --/CR res: [0x200000401:0x268:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764770.340935:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966906880:192.168.202.146@tcp:101:dbench.0 00000100:00100000:0.0:1775764770.341107:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16fd80 going to sleep for 11 seconds 00000100:00100000:1.0:1775764770.341246:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966906368:192.168.202.146@tcp:3:dbench.0 00010000:00010000:1.0:1775764770.373188:0:19849:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c1a3 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x269:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19849 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.373196:0:19849:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c1a3 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x269:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19849 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.373220:0:19849:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9800/0x36f9bdc9e41cb202 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ef946 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764770.373229:0:19849:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9800/0x36f9bdc9e41cb202 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ef946 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00010000:1.0:1775764770.373236:0:19849:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e41cb202 from it ffffafbac1e6fd48 00010000:00010000:1.0:1775764770.373240:0:19849:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9800/0x36f9bdc9e41cb202 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e4ef946 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764770.373252:0:19849:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9800/0x36f9bdc9e41cb202 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ef946 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764770.373274:0:19849:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be800/0x36f9bdc9e421c1aa lrc: 3/1,0 mode: --/CR res: [0x200000401:0x269:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 19849 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.373280:0:19849:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be800/0x36f9bdc9e421c1aa lrc: 3/1,0 mode: --/CR res: [0x200000401:0x269:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19849 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.373296:0:19849:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bfc00/0x36f9bdc9e421c1b1 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x269:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 19849 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.373302:0:19849:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bfc00/0x36f9bdc9e421c1b1 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x269:0x0].0x0 bits 0x3/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19849 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.373317:0:19849:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bea00/0x36f9bdc9e421c1b8 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x269:0x0].0x0 bits 0x0/0x0 rrc: 6 type: IBT flags: 0x10000000000000 pid: 19849 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.373324:0:19849:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bea00/0x36f9bdc9e421c1b8 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x269:0x0].0x0 bits 0x3/0x0 rrc: 6 type: IBT flags: 0x0 pid: 19849 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764770.373416:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970c02d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966907520:192.168.202.146@tcp:63:ll_sa.0 00010000:00010000:0.0:1775764770.381936:0:2379:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c1a3 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x269:0x0].0x0 bits 0x3/0x0 rrc: 6 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f88fa expref: -99 pid: 19849 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.381979:0:2379:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c1a3 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x270:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f88fa expref: -99 pid: 19849 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.381988:0:2379:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c1a3 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x270:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f88fa expref: -99 pid: 19849 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.382000:0:2379:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c1a3 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x270:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f88fa expref: -99 pid: 19849 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.382012:0:2379:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c1a3 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x270:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f88fa expref: -99 pid: 19849 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.382024:0:2379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c1a3 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x270:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f88fa expref: -99 pid: 19849 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.382032:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c1a3 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x270:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f88fa expref: -99 pid: 19849 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.382039:0:2379:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c1a3 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x270:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f88fa expref: -99 pid: 19849 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764770.382053:0:2379:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970c02d80 x1862022966907520/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 1872/3176 e 0 to 0 dl 1775764786 ref 1 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1775764770.382065:0:2379:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c1a3 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x270:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f88fa expref: -99 pid: 19849 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1775764770.382078:0:2379:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c1a3 from it ffff8f9970e0b188 00010000:00010000:0.0:1775764770.382082:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c1a3 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x270:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f88fa expref: -99 pid: 19849 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.382091:0:2379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c1a3 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x270:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f88fa expref: -99 pid: 19849 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.382130:0:2379:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be800/0x36f9bdc9e421c1aa lrc: 4/1,0 mode: --/CR res: [0x200000401:0x269:0x0].0x0 bits 0x3/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8916 expref: -99 pid: 19849 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.382142:0:2379:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be800/0x36f9bdc9e421c1aa lrc: 4/1,0 mode: --/PR res: [0x200000401:0x276:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8916 expref: -99 pid: 19849 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.382149:0:2379:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be800/0x36f9bdc9e421c1aa lrc: 4/1,0 mode: --/PR res: [0x200000401:0x276:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8916 expref: -99 pid: 19849 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.382162:0:2379:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be800/0x36f9bdc9e421c1aa lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x276:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8916 expref: -99 pid: 19849 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.382170:0:2379:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be800/0x36f9bdc9e421c1aa lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x276:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8916 expref: -99 pid: 19849 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.382178:0:2379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be800/0x36f9bdc9e421c1aa lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x276:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8916 expref: -99 pid: 19849 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.382185:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be800/0x36f9bdc9e421c1aa lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x276:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8916 expref: -99 pid: 19849 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.382193:0:2379:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be800/0x36f9bdc9e421c1aa lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x276:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8916 expref: -99 pid: 19849 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764770.382202:0:2379:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970c02d80 x1862022966907520/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 1872/3176 e 0 to 0 dl 1775764786 ref 2 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1775764770.382211:0:2379:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be800/0x36f9bdc9e421c1aa lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x276:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8916 expref: -99 pid: 19849 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1775764770.382220:0:2379:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c1aa from it ffff8f9970e0bd88 00010000:00010000:0.0:1775764770.382223:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be800/0x36f9bdc9e421c1aa lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x276:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8916 expref: -99 pid: 19849 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.382230:0:2379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be800/0x36f9bdc9e421c1aa lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x276:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8916 expref: -99 pid: 19849 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.382248:0:2379:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bfc00/0x36f9bdc9e421c1b1 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x269:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8932 expref: -99 pid: 19849 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.382258:0:2379:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bfc00/0x36f9bdc9e421c1b1 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x274:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8932 expref: -99 pid: 19849 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.382266:0:2379:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bfc00/0x36f9bdc9e421c1b1 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x274:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8932 expref: -99 pid: 19849 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.382285:0:2379:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bfc00/0x36f9bdc9e421c1b1 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x274:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8932 expref: -99 pid: 19849 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.382293:0:2379:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bfc00/0x36f9bdc9e421c1b1 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x274:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8932 expref: -99 pid: 19849 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.382300:0:2379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bfc00/0x36f9bdc9e421c1b1 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x274:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8932 expref: -99 pid: 19849 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.382307:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bfc00/0x36f9bdc9e421c1b1 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x274:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8932 expref: -99 pid: 19849 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.382315:0:2379:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bfc00/0x36f9bdc9e421c1b1 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x274:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8932 expref: -99 pid: 19849 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764770.382323:0:2379:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970c02d80 x1862022966907520/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 1872/3176 e 0 to 0 dl 1775764786 ref 3 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1775764770.382339:0:2379:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bfc00/0x36f9bdc9e421c1b1 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x274:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8932 expref: -99 pid: 19849 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1775764770.382348:0:2379:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c1b1 from it ffff8f9970e09588 00010000:00010000:0.0:1775764770.382350:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bfc00/0x36f9bdc9e421c1b1 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x274:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8932 expref: -99 pid: 19849 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.382357:0:2379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bfc00/0x36f9bdc9e421c1b1 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x274:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8932 expref: -99 pid: 19849 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.382371:0:2379:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bea00/0x36f9bdc9e421c1b8 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x269:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f894e expref: -99 pid: 19849 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.382383:0:2379:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bea00/0x36f9bdc9e421c1b8 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x26d:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f894e expref: -99 pid: 19849 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.382920:0:2379:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bea00/0x36f9bdc9e421c1b8 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x26d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f894e expref: -99 pid: 19849 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.382931:0:2379:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bea00/0x36f9bdc9e421c1b8 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x26d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f894e expref: -99 pid: 19849 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.382939:0:2379:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bea00/0x36f9bdc9e421c1b8 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x26d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f894e expref: -99 pid: 19849 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.382960:0:2379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bea00/0x36f9bdc9e421c1b8 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x26d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f894e expref: -99 pid: 19849 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.382971:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bea00/0x36f9bdc9e421c1b8 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x26d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f894e expref: -99 pid: 19849 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.382977:0:2379:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bea00/0x36f9bdc9e421c1b8 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x26d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f894e expref: -99 pid: 19849 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764770.382986:0:2379:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970c02d80 x1862022966907520/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 1872/3176 e 0 to 0 dl 1775764786 ref 4 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1775764770.382994:0:2379:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bea00/0x36f9bdc9e421c1b8 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f894e expref: -99 pid: 19849 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1775764770.383004:0:2379:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c1b8 from it ffff8f9970e08588 00010000:00010000:0.0:1775764770.383007:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bea00/0x36f9bdc9e421c1b8 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f894e expref: -99 pid: 19849 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.383013:0:2379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bea00/0x36f9bdc9e421c1b8 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f894e expref: -99 pid: 19849 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1775764770.383032:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970c02d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966907520:192.168.202.146@tcp:63:ll_sa.0 00010000:00010000:0.0:1775764770.383068:0:131:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c1a3 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x270:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f88fa expref: -99 pid: 19849 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764770.383181:0:131:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x270:0x0](ffff8f9952e67448): rc = 0 00000080:00200000:0.0:1775764770.383202:0:131:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x270:0x0]: layout version change: 4294967294 -> 3 00000080:00010000:0.0:1775764770.383212:0:131:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x270:0x0](ffff8f9952e67448) for lock 0x36f9bdc9e421c1a3 00000080:00010000:0.0:1775764770.383226:0:131:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c1a3 from it ffff8f9970e0b188 00010000:00010000:0.0:1775764770.383230:0:131:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c1a3 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x270:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f88fa expref: -99 pid: 19849 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.383238:0:131:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c1a3 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x270:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f88fa expref: -99 pid: 19849 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764770.383258:0:131:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be800/0x36f9bdc9e421c1aa lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x276:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8916 expref: -99 pid: 19849 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764770.383286:0:131:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x276:0x0](ffff8f9952e61148): rc = 0 00000080:00200000:0.0:1775764770.383300:0:131:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x276:0x0]: layout version change: 4294967294 -> 3 00000080:00010000:0.0:1775764770.383308:0:131:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x276:0x0](ffff8f9952e61148) for lock 0x36f9bdc9e421c1aa 00000080:00010000:0.0:1775764770.383311:0:131:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c1aa from it ffff8f9970e0bd88 00010000:00010000:0.0:1775764770.383314:0:131:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be800/0x36f9bdc9e421c1aa lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x276:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8916 expref: -99 pid: 19849 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.383322:0:131:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be800/0x36f9bdc9e421c1aa lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x276:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8916 expref: -99 pid: 19849 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764770.383335:0:131:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bfc00/0x36f9bdc9e421c1b1 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x274:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8932 expref: -99 pid: 19849 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764770.383364:0:131:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x274:0x0](ffff8f9952da5b88): rc = 0 00000080:00200000:0.0:1775764770.383377:0:131:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x274:0x0]: layout version change: 4294967294 -> 3 00000080:00010000:0.0:1775764770.383385:0:131:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x274:0x0](ffff8f9952da5b88) for lock 0x36f9bdc9e421c1b1 00000080:00010000:0.0:1775764770.384305:0:131:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c1b1 from it ffff8f9970e09588 00010000:00010000:0.0:1775764770.384311:0:131:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bfc00/0x36f9bdc9e421c1b1 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x274:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8932 expref: -99 pid: 19849 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.384320:0:131:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bfc00/0x36f9bdc9e421c1b1 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x274:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8932 expref: -99 pid: 19849 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764770.384342:0:131:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bea00/0x36f9bdc9e421c1b8 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f894e expref: -99 pid: 19849 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764770.384381:0:131:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x26d:0x0](ffff8f9952e65b88): rc = 0 00000080:00200000:0.0:1775764770.385015:0:131:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x26d:0x0]: layout version change: 4294967294 -> 3 00000080:00010000:0.0:1775764770.385031:0:131:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x26d:0x0](ffff8f9952e65b88) for lock 0x36f9bdc9e421c1b8 00000080:00010000:0.0:1775764770.385085:0:131:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c1b8 from it ffff8f9970e08588 00010000:00010000:0.0:1775764770.385089:0:131:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bea00/0x36f9bdc9e421c1b8 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f894e expref: -99 pid: 19849 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.385098:0:131:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bea00/0x36f9bdc9e421c1b8 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f894e expref: -99 pid: 19849 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764770.408419:0:19851:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf600/0x36f9bdc9e421c1e9 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26a:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19851 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.408428:0:19851:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf600/0x36f9bdc9e421c1e9 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26a:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19851 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.408446:0:19851:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be000/0x36f9bdc9e421c1f0 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26a:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 19851 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.408453:0:19851:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be000/0x36f9bdc9e421c1f0 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26a:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19851 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.408466:0:19851:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be400/0x36f9bdc9e421c1f7 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26a:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 19851 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.408472:0:19851:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be400/0x36f9bdc9e421c1f7 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26a:0x0].0x0 bits 0x3/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19851 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.408489:0:19851:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764770.408497:0:19851:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00010000:1.0:1775764770.408504:0:19851:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e41cb2b1 from it ffffafbac1d7bd48 00010000:00010000:1.0:1775764770.408507:0:19851:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764770.408514:0:19851:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764770.408538:0:19851:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c1fe lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26a:0x0].0x0 bits 0x0/0x0 rrc: 6 type: IBT flags: 0x10000000000000 pid: 19851 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.408544:0:19851:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c1fe lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26a:0x0].0x0 bits 0x3/0x0 rrc: 6 type: IBT flags: 0x0 pid: 19851 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764770.408636:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970c03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966908672:192.168.202.146@tcp:63:ll_sa.0 00010000:00010000:1.0:1775764770.422883:0:2380:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf600/0x36f9bdc9e421c1e9 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x26a:0x0].0x0 bits 0x3/0x0 rrc: 6 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f89c5 expref: -99 pid: 19851 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.422911:0:2380:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf600/0x36f9bdc9e421c1e9 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x273:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f89c5 expref: -99 pid: 19851 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.422919:0:2380:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf600/0x36f9bdc9e421c1e9 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x273:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f89c5 expref: -99 pid: 19851 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.422930:0:2380:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf600/0x36f9bdc9e421c1e9 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x273:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f89c5 expref: -99 pid: 19851 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.422942:0:2380:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf600/0x36f9bdc9e421c1e9 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x273:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f89c5 expref: -99 pid: 19851 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.422966:0:2380:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf600/0x36f9bdc9e421c1e9 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x273:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f89c5 expref: -99 pid: 19851 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.422974:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf600/0x36f9bdc9e421c1e9 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x273:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f89c5 expref: -99 pid: 19851 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.422981:0:2380:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf600/0x36f9bdc9e421c1e9 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x273:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f89c5 expref: -99 pid: 19851 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764770.422996:0:2380:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970c03b80 x1862022966908672/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 1872/3176 e 0 to 0 dl 1775764786 ref 1 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:1.0:1775764770.423007:0:2380:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf600/0x36f9bdc9e421c1e9 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x273:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f89c5 expref: -99 pid: 19851 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:1.0:1775764770.423020:0:2380:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c1e9 from it ffff8f9970e0a188 00010000:00010000:1.0:1775764770.423023:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf600/0x36f9bdc9e421c1e9 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x273:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f89c5 expref: -99 pid: 19851 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.423031:0:2380:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf600/0x36f9bdc9e421c1e9 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x273:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f89c5 expref: -99 pid: 19851 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.423058:0:2380:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be000/0x36f9bdc9e421c1f0 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x26a:0x0].0x0 bits 0x3/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f89e1 expref: -99 pid: 19851 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.423068:0:2380:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be000/0x36f9bdc9e421c1f0 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x26e:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f89e1 expref: -99 pid: 19851 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.423075:0:2380:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be000/0x36f9bdc9e421c1f0 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x26e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f89e1 expref: -99 pid: 19851 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.423081:0:2380:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be000/0x36f9bdc9e421c1f0 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x26e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f89e1 expref: -99 pid: 19851 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.423088:0:2380:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be000/0x36f9bdc9e421c1f0 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x26e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f89e1 expref: -99 pid: 19851 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.423095:0:2380:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be000/0x36f9bdc9e421c1f0 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x26e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f89e1 expref: -99 pid: 19851 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.423102:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be000/0x36f9bdc9e421c1f0 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x26e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f89e1 expref: -99 pid: 19851 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.423113:0:2380:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be000/0x36f9bdc9e421c1f0 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x26e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f89e1 expref: -99 pid: 19851 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764770.423134:0:2380:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970c03b80 x1862022966908672/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 1872/3176 e 0 to 0 dl 1775764786 ref 2 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:1.0:1775764770.423143:0:2380:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be000/0x36f9bdc9e421c1f0 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f89e1 expref: -99 pid: 19851 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:1.0:1775764770.423151:0:2380:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c1f0 from it ffff8f9970e0a588 00010000:00010000:1.0:1775764770.423154:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be000/0x36f9bdc9e421c1f0 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f89e1 expref: -99 pid: 19851 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.423161:0:2380:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be000/0x36f9bdc9e421c1f0 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f89e1 expref: -99 pid: 19851 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.423174:0:2380:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be400/0x36f9bdc9e421c1f7 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x26a:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8a04 expref: -99 pid: 19851 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.423184:0:2380:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be400/0x36f9bdc9e421c1f7 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x271:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8a04 expref: -99 pid: 19851 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.423190:0:2380:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be400/0x36f9bdc9e421c1f7 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x271:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8a04 expref: -99 pid: 19851 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.423197:0:2380:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be400/0x36f9bdc9e421c1f7 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x271:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8a04 expref: -99 pid: 19851 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.423208:0:2380:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be400/0x36f9bdc9e421c1f7 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x271:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8a04 expref: -99 pid: 19851 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.423217:0:2380:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be400/0x36f9bdc9e421c1f7 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x271:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8a04 expref: -99 pid: 19851 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.423223:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be400/0x36f9bdc9e421c1f7 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x271:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8a04 expref: -99 pid: 19851 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.423229:0:2380:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be400/0x36f9bdc9e421c1f7 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x271:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8a04 expref: -99 pid: 19851 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764770.423236:0:2380:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970c03b80 x1862022966908672/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 1872/3176 e 0 to 0 dl 1775764786 ref 3 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:1.0:1775764770.423244:0:2380:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be400/0x36f9bdc9e421c1f7 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x271:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8a04 expref: -99 pid: 19851 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:1.0:1775764770.423251:0:2380:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c1f7 from it ffff8f9970e09188 00010000:00010000:1.0:1775764770.423253:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be400/0x36f9bdc9e421c1f7 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x271:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8a04 expref: -99 pid: 19851 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.423260:0:2380:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be400/0x36f9bdc9e421c1f7 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x271:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8a04 expref: -99 pid: 19851 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.423271:0:2380:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c1fe lrc: 4/1,0 mode: --/CR res: [0x200000401:0x26a:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8a20 expref: -99 pid: 19851 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.423286:0:2380:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c1fe lrc: 4/1,0 mode: --/PR res: [0x200000401:0x26f:0x0].0x0 bits 0x3/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8a20 expref: -99 pid: 19851 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.423291:0:2380:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c1fe lrc: 4/1,0 mode: --/PR res: [0x200000401:0x26f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8a20 expref: -99 pid: 19851 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.423298:0:2380:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c1fe lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x26f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8a20 expref: -99 pid: 19851 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.423305:0:2380:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c1fe lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x26f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8a20 expref: -99 pid: 19851 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.423312:0:2380:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c1fe lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x26f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8a20 expref: -99 pid: 19851 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.423319:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c1fe lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x26f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8a20 expref: -99 pid: 19851 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.423325:0:2380:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c1fe lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x26f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8a20 expref: -99 pid: 19851 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764770.423333:0:2380:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970c03b80 x1862022966908672/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 1872/3176 e 0 to 0 dl 1775764786 ref 4 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:1.0:1775764770.423340:0:2380:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c1fe lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8a20 expref: -99 pid: 19851 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:1.0:1775764770.423353:0:2380:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c1fe from it ffff8f9970e0a988 00010000:00010000:1.0:1775764770.423355:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c1fe lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8a20 expref: -99 pid: 19851 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.423361:0:2380:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c1fe lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8a20 expref: -99 pid: 19851 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000100:00100000:1.0:1775764770.423375:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970c03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966908672:192.168.202.146@tcp:63:ll_sa.0 00010000:00010000:1.0:1775764770.423409:0:21:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf600/0x36f9bdc9e421c1e9 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x273:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f89c5 expref: -99 pid: 19851 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764770.423487:0:21:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x273:0x0](ffff8f9952db7448): rc = 0 00000080:00200000:1.0:1775764770.423507:0:21:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x273:0x0]: layout version change: 4294967294 -> 3 00000080:00010000:1.0:1775764770.423516:0:21:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x273:0x0](ffff8f9952db7448) for lock 0x36f9bdc9e421c1e9 00000080:00010000:1.0:1775764770.423532:0:21:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c1e9 from it ffff8f9970e0a188 00010000:00010000:1.0:1775764770.423536:0:21:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf600/0x36f9bdc9e421c1e9 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x273:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f89c5 expref: -99 pid: 19851 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.423543:0:21:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf600/0x36f9bdc9e421c1e9 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x273:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f89c5 expref: -99 pid: 19851 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764770.423561:0:21:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be000/0x36f9bdc9e421c1f0 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f89e1 expref: -99 pid: 19851 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764770.423593:0:21:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x26e:0x0](ffff8f9952db0908): rc = 0 00000080:00200000:1.0:1775764770.423620:0:21:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x26e:0x0]: layout version change: 4294967294 -> 3 00000080:00010000:1.0:1775764770.423640:0:21:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x26e:0x0](ffff8f9952db0908) for lock 0x36f9bdc9e421c1f0 00000080:00010000:1.0:1775764770.423644:0:21:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c1f0 from it ffff8f9970e0a588 00010000:00010000:1.0:1775764770.423647:0:21:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be000/0x36f9bdc9e421c1f0 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f89e1 expref: -99 pid: 19851 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.423654:0:21:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be000/0x36f9bdc9e421c1f0 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f89e1 expref: -99 pid: 19851 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764770.423668:0:21:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be400/0x36f9bdc9e421c1f7 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x271:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8a04 expref: -99 pid: 19851 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764770.423699:0:21:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x271:0x0](ffff8f9952da63c8): rc = 0 00000080:00200000:1.0:1775764770.423713:0:21:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x271:0x0]: layout version change: 4294967294 -> 3 00000080:00010000:1.0:1775764770.423720:0:21:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x271:0x0](ffff8f9952da63c8) for lock 0x36f9bdc9e421c1f7 00000080:00010000:1.0:1775764770.423723:0:21:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c1f7 from it ffff8f9970e09188 00010000:00010000:1.0:1775764770.423726:0:21:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be400/0x36f9bdc9e421c1f7 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x271:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8a04 expref: -99 pid: 19851 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.423733:0:21:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be400/0x36f9bdc9e421c1f7 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x271:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8a04 expref: -99 pid: 19851 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764770.423788:0:21:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c1fe lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8a20 expref: -99 pid: 19851 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764770.423818:0:21:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x26f:0x0](ffff8f9952e621c8): rc = 0 00000080:00200000:1.0:1775764770.423830:0:21:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x26f:0x0]: layout version change: 4294967294 -> 3 00000080:00010000:1.0:1775764770.423838:0:21:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x26f:0x0](ffff8f9952e621c8) for lock 0x36f9bdc9e421c1fe 00000080:00010000:1.0:1775764770.423841:0:21:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c1fe from it ffff8f9970e0a988 00010000:00010000:1.0:1775764770.423845:0:21:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c1fe lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8a20 expref: -99 pid: 19851 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.423852:0:21:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c1fe lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8a20 expref: -99 pid: 19851 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000020:00010000:0.0:1775764770.462648:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99709423f8 (ffff8f994ef58cf8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764770.462652:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99709423f8 (ffff8f994ef58cf8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.462656:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994166fbb0 (ffff8f994ef587e8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764770.462667:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4f14cf5/0x36f9bdc9e4211ec0 lrc: 4/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1232896->1236991) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00010000:00010000:0.0:1775764770.462683:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4f14cf5/0x36f9bdc9e4211ec0 lrc: 3/0,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1232896->1236991) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 0 00000020:00010000:0.0:1775764770.462698:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99709423f8 (ffff8f994ef58cf8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764770.462702:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994166fbb0 (ffff8f994ef58cf8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764770.462732:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x266:0x0] ignore/verify layout 0/0, layout version 4 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764770.462951:0:19724:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8) pos/size0/7168 32bit_api 0 00000080:00200000:0.0:1775764770.463001:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.463006:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.463013:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:0.0:1775764770.463020:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.463024:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:0.0:1775764770.463027:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.463030:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:0.0:1775764770.463032:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.463035:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=SEED, flags=1 00000080:00200008:0.0:1775764770.463044:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file SEED:[0x200000401:0x2e5:0x0](ffff8f9952dd2a08), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764770.465159:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2e5:0x0](ffff8f9952dd2a08),name=SEED 00000002:00010000:0.0:1775764770.465186:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2e5:0x0]) in obj [0x200000401:0x26b:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764770.465204:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3b8600/0x36f9bdc9e41cf905 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4fd843 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764770.465218:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3b8600/0x36f9bdc9e41cf905 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4fd843 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:0.0:1775764770.465226:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26b:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764770.465232:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2e5:0x0](ffff8f9952dd2a08) 00000080:00010000:0.0:1775764770.465236:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2e5:0x0](ffff8f9952dd2a08) for lock 0x36f9bdc9e41cf905 00000080:00010000:0.0:1775764770.465241:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e41cf905 from it ffffafbac1c5bcb8 00010000:00010000:0.0:1775764770.465245:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3b8600/0x36f9bdc9e41cf905 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e4fd843 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764770.465258:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3b8600/0x36f9bdc9e41cf905 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4fd843 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00000080:00200000:0.0:1775764770.465302:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.465307:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.465312:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=19 00000080:00200000:0.0:1775764770.465317:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.465321:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=19 00000080:00200000:0.0:1775764770.465324:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.465327:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=19 00000080:00200000:0.0:1775764770.465329:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.465332:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=SEED, flags=259 00000080:00200000:0.0:1775764770.465340:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2e5:0x0](ffff8f9952dd2a08), inode mode 41c0 mask 44 00000080:00200008:0.0:1775764770.465348:0:19724:0:(file.c:1057:ll_file_open()) START file SEED:[0x200000401:0x2e5:0x0](ffff8f9952dd2a08), flags 304000 00000080:00200008:0.0:1775764770.465364:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file SEED:[0x200000401:0x2e5:0x0](ffff8f9952dd2a08), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764770.465368:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2e5:0x0](ffff8f9952dd2a08),name=SEED 00000002:00010000:0.0:1775764770.465375:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2e5:0x0]) in obj [0x200000401:0x26b:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764770.465382:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3b8600/0x36f9bdc9e41cf905 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4fd843 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00010000:00010000:0.0:1775764770.465402:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3b8600/0x36f9bdc9e41cf905 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4fd843 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00000080:00200000:0.0:1775764770.465408:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26b:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764770.465412:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2e5:0x0](ffff8f9952dd2a08) 00000080:00010000:0.0:1775764770.465414:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2e5:0x0](ffff8f9952dd2a08) for lock 0x36f9bdc9e41cf905 00000080:00010000:0.0:1775764770.465417:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e41cf905 from it ffffafbac1c5bcd8 00010000:00010000:0.0:1775764770.465421:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3b8600/0x36f9bdc9e41cf905 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e4fd843 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00010000:00010000:0.0:1775764770.465428:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3b8600/0x36f9bdc9e41cf905 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4fd843 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00000080:00200000:0.0:1775764770.465484:0:19724:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2e5:0x0](ffff8f9952dd2a08) pos/size0/7168 32bit_api 0 00000080:00200000:0.0:1775764770.465518:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.465521:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.465524:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:0.0:1775764770.465527:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.465529:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:0.0:1775764770.465531:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.465534:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:0.0:1775764770.465536:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.465539:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=SEED, flags=17 00000080:00200000:0.0:1775764770.465541:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2e5:0x0](ffff8f9952dd2a08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.465552:0:19724:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=LARGE.FIL, dir=[0x200000401:0x2e5:0x0](ffff8f9952dd2a08), flags=1 00000080:00200000:0.0:1775764770.465556:0:19724:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=LARGE.FIL, dir=[0x200000401:0x2e5:0x0](ffff8f9952dd2a08), intent=getattr 00000002:00010000:0.0:1775764770.465566:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: LARGE.FIL,[0x0:0x0:0x0]) in obj [0x200000401:0x2e5:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764770.465630:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c22f lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2e5:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.465639:0:19724:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c22f lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2e5:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.465648:0:19724:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c22f lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2e5:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764770.465672:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966910464:192.168.202.146@tcp:101:dbench.0 00000100:00100000:0.0:1775764770.465795:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f240 going to sleep for 11 seconds 00010000:00010000:1.0:1775764770.466461:0:19853:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/696 (0 0) 00010000:00010000:1.0:1775764770.466494:0:19853:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be200/0x36f9bdc9e421c236 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 19853 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.466501:0:19853:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be200/0x36f9bdc9e421c236 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19853 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.466522:0:19853:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/641 (0 0) 00010000:00010000:1.0:1775764770.466531:0:19853:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf000/0x36f9bdc9e421c23d lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 19853 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.466537:0:19853:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf000/0x36f9bdc9e421c23d lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x3/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19853 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.466551:0:19853:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/632 (0 0) 00010000:00010000:1.0:1775764770.466559:0:19853:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bec00/0x36f9bdc9e421c244 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x0/0x0 rrc: 6 type: IBT flags: 0x10000000000000 pid: 19853 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.466564:0:19853:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bec00/0x36f9bdc9e421c244 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x3/0x0 rrc: 6 type: IBT flags: 0x0 pid: 19853 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.466577:0:19853:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/647 (0 0) 00010000:00010000:1.0:1775764770.466585:0:19853:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c24b lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x0/0x0 rrc: 7 type: IBT flags: 0x10000000000000 pid: 19853 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.466595:0:19853:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c24b lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x3/0x0 rrc: 7 type: IBT flags: 0x0 pid: 19853 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.466922:0:19853:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/681 (0 0) 00010000:00010000:1.0:1775764770.466937:0:19853:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480400/0x36f9bdc9e421c252 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x0/0x0 rrc: 8 type: IBT flags: 0x10000000000000 pid: 19853 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.466943:0:19853:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480400/0x36f9bdc9e421c252 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x3/0x0 rrc: 8 type: IBT flags: 0x0 pid: 19853 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.466971:0:19853:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/689 (0 0) 00010000:00010000:1.0:1775764770.466981:0:19853:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941481c00/0x36f9bdc9e421c259 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x0/0x0 rrc: 9 type: IBT flags: 0x10000000000000 pid: 19853 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.466986:0:19853:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941481c00/0x36f9bdc9e421c259 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x3/0x0 rrc: 9 type: IBT flags: 0x0 pid: 19853 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.466999:0:19853:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/658 (0 0) 00010000:00010000:1.0:1775764770.467006:0:19853:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c260 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x0/0x0 rrc: 10 type: IBT flags: 0x10000000000000 pid: 19853 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.467012:0:19853:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c260 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x3/0x0 rrc: 10 type: IBT flags: 0x0 pid: 19853 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.467026:0:19853:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/737 (0 0) 00010000:00010000:1.0:1775764770.467036:0:19853:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5c00/0x36f9bdc9e421c267 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x0/0x0 rrc: 11 type: IBT flags: 0x10000000000000 pid: 19853 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.467042:0:19853:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5c00/0x36f9bdc9e421c267 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x3/0x0 rrc: 11 type: IBT flags: 0x0 pid: 19853 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764770.467222:0:2380:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1775764770.467250:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970c03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966910592:192.168.202.146@tcp:63:ll_sa.0 00010000:00010000:0.0:1775764770.473772:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1775764770.473780:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c22f lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ec:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f8acf expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.473789:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99435d4c00 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:0.0:1775764770.473794:0:5571:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c22f lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ec:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f8acf expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.473802:0:5571:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c22f lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ec:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f8acf expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.473811:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c22f lrc: 3/0,0 mode: PR/PR res: [0x200000401:0x2ec:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f8acf expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.473829:0:5571:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966bb1000/0x36f9bdc9e421a174 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ec:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f27a9 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764770.473868:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c22f lrc: 2/0,0 mode: --/PR res: [0x200000401:0x2ec:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8acf expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.473875:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764770.473899:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c22f lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2ec:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8acf expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.473905:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c22f lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2ec:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8acf expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.473918:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764770.473921:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764770.473923:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764770.473930:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1775764770.473945:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966910720:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764770.486531:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703dce00 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966910336:192.168.202.146@tcp:36:dbench.0 00000080:00200000:1.0:1775764770.486631:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x273:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764770.486644:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:1.0:1775764770.486664:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:1.0:1775764770.486665:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:1.0:1775764770.486698:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x273:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764770.486712:0:19723:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x273:0x0](ffff8f9952db7448) 00000008:00010000:1.0:1775764770.486765:0:19723:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 1048576 blocks 2051 [cma]time: 1775764050 1775764050 0 by oinfo size 1048576 blocks 2051 [cma]time 1775764050 1775764050 0 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000031510ace/0x36f9bdc9e421c20c lrc: 1/0,0 mode: PR/PR res: [0x240000403:0x37:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5f8a27 expref: -99 pid: 19852 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764770.486842:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.486847:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.486853:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:1.0:1775764770.486858:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.486860:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:1.0:1775764770.486863:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.486873:0:19723:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=filler.000, dir=[0x200000401:0x26a:0x0](ffff8f9952da00c8), flags=1 00000080:00200000:1.0:1775764770.486877:0:19723:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=filler.000, dir=[0x200000401:0x26a:0x0](ffff8f9952da00c8), intent=getattr 00010000:00010000:1.0:1775764770.486894:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be000/0x36f9bdc9e421c1f0 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f89e1 expref: -99 pid: 19851 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:1.0:1775764770.486909:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c1f0 from it ffffafbac1c7b900 00010000:00010000:1.0:1775764770.486913:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be000/0x36f9bdc9e421c1f0 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f89e1 expref: -99 pid: 19851 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764770.486921:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be000/0x36f9bdc9e421c1f0 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f89e1 expref: -99 pid: 19851 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:1.0:1775764770.486961:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file filler.000:[0x200000401:0x26e:0x0](ffff8f9952db0908), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764770.486966:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x26e:0x0](ffff8f9952db0908),name=filler.000 00000002:00010000:1.0:1775764770.486983:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x26e:0x0]) in obj [0x200000401:0x26a:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764770.486991:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be000/0x36f9bdc9e421c1f0 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f89e1 expref: -99 pid: 19851 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764770.487001:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be000/0x36f9bdc9e421c1f0 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f89e1 expref: -99 pid: 19851 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764770.487008:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26a:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764770.487013:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x26e:0x0](ffff8f9952db0908) 00000080:00010000:1.0:1775764770.487016:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x26e:0x0](ffff8f9952db0908) for lock 0x36f9bdc9e421c1f0 00000080:00010000:1.0:1775764770.487021:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c1f0 from it ffffafbac1c7bcb8 00010000:00010000:1.0:1775764770.487023:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be000/0x36f9bdc9e421c1f0 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f89e1 expref: -99 pid: 19851 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764770.487030:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be000/0x36f9bdc9e421c1f0 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f89e1 expref: -99 pid: 19851 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764770.487051:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x26e:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764770.487058:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x26e:0x0] 00000008:00010000:1.0:1775764770.487076:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764770.487079:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99705ee1b0, osc lock ffff8f995875db50, flags 0x400001002 00010000:00010000:1.0:1775764770.487095:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000003c3894b8/0x36f9bdc9e421c213 lrc: 2/1,0 mode: PR/PR res: [0x280000403:0x34:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8a2e expref: -99 pid: 19852 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.487105:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000003c3894b8/0x36f9bdc9e421c213 lrc: 2/1,0 mode: PR/PR res: [0x280000403:0x34:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8a2e expref: -99 pid: 19852 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:1.0:1775764770.487128:0:19723:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 1048576, setting rss=1048576; kms=1048576, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000003c3894b8/0x36f9bdc9e421c213 lrc: 4/1,0 mode: PR/PR res: [0x280000403:0x34:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8a2e expref: -99 pid: 19852 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:1.0:1775764770.487138:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764770.487142:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000003c3894b8/0x36f9bdc9e421c213 lrc: 5/1,0 mode: PR/PR res: [0x280000403:0x34:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5f8a2e expref: -99 pid: 19852 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.487148:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000003c3894b8/0x36f9bdc9e421c213 lrc: 4/0,0 mode: PR/PR res: [0x280000403:0x34:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5f8a2e expref: -99 pid: 19852 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:1.0:1775764770.487158:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764770.487166:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x26e:0x0] updating i_size 1048576 i_blocks 2051 00000020:00010000:1.0:1775764770.487170:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_release():245 00000020:00010000:1.0:1775764770.487172:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764770.487175:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705ee1b0 (ffff8f99705b8ea8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764770.487179:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764770.487181:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705ee1b0 (ffff8f99705b8128/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764770.487194:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x26e:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764770.487212:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.487215:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.487218:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:1.0:1775764770.487222:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.487224:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=16 00000080:00200000:1.0:1775764770.487226:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.487231:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=filler.000, flags=0 00000080:00200000:1.0:1775764770.487233:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 3 00000080:00200000:1.0:1775764770.487238:0:19723:0:(namei.c:2395:ll_unlink()) VFS Op:name=filler.000, dir=[0x200000401:0x26a:0x0](ffff8f9952da00c8) 00010000:00010000:1.0:1775764770.487271:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be000/0x36f9bdc9e421c1f0 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f89e1 expref: -99 pid: 19851 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764770.487292:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x26e:0x0]: losing layout lock 00010000:00010000:1.0:1775764770.487333:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be000/0x36f9bdc9e421c1f0 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x26e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f89e1 expref: -99 pid: 19851 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764770.487340:0:19723:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764770.487342:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be000/0x36f9bdc9e421c1f0 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x26e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f89e1 expref: -99 pid: 19851 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000020:00100000:1.0:1775764770.487362:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1775764770.487370:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970c03480 x1862022966911360/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764770.487390:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970c03480 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966911360:192.168.202.146@tcp:36:dbench.0 00000100:00100000:1.0:1775764770.487444:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f997130db40 going to sleep for 11 seconds 00000100:00100000:0.0:1775764770.489584:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966910720:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1775764770.502057:0:2380:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be200/0x36f9bdc9e421c236 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x3/0x0 rrc: 11 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b00 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502085:0:2380:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be200/0x36f9bdc9e421c236 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b00 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502093:0:2380:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be200/0x36f9bdc9e421c236 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b00 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502105:0:2380:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be200/0x36f9bdc9e421c236 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b00 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502127:0:2380:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be200/0x36f9bdc9e421c236 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b00 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502138:0:2380:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be200/0x36f9bdc9e421c236 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b00 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502146:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be200/0x36f9bdc9e421c236 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b00 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502153:0:2380:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be200/0x36f9bdc9e421c236 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b00 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764770.502172:0:2380:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970c03b80 x1862022966910592/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 3408/3368 e 0 to 0 dl 1775764786 ref 1 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000080:00010000:0.0:1775764770.502185:0:2380:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c236 from it ffff8f9970e0a188 00010000:00010000:0.0:1775764770.502188:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be200/0x36f9bdc9e421c236 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b00 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502195:0:2380:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be200/0x36f9bdc9e421c236 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b00 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.502226:0:2380:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf000/0x36f9bdc9e421c23d lrc: 4/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x3/0x0 rrc: 10 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b1c expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502235:0:2380:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf000/0x36f9bdc9e421c23d lrc: 4/1,0 mode: --/PR res: [0x200000401:0x281:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b1c expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502241:0:2380:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf000/0x36f9bdc9e421c23d lrc: 4/1,0 mode: --/PR res: [0x200000401:0x281:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b1c expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502249:0:2380:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf000/0x36f9bdc9e421c23d lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x281:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b1c expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502256:0:2380:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf000/0x36f9bdc9e421c23d lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x281:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b1c expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502262:0:2380:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf000/0x36f9bdc9e421c23d lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x281:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b1c expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502272:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf000/0x36f9bdc9e421c23d lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x281:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b1c expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502278:0:2380:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf000/0x36f9bdc9e421c23d lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x281:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b1c expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764770.502286:0:2380:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970c03b80 x1862022966910592/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 3408/3368 e 0 to 0 dl 1775764786 ref 2 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000080:00010000:0.0:1775764770.502294:0:2380:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c23d from it ffff8f9970e09988 00010000:00010000:0.0:1775764770.502296:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf000/0x36f9bdc9e421c23d lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x281:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b1c expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502302:0:2380:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf000/0x36f9bdc9e421c23d lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x281:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b1c expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.502315:0:2380:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bec00/0x36f9bdc9e421c244 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x3/0x0 rrc: 9 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b38 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502323:0:2380:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bec00/0x36f9bdc9e421c244 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x278:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b38 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502329:0:2380:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bec00/0x36f9bdc9e421c244 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x278:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b38 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502336:0:2380:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bec00/0x36f9bdc9e421c244 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x278:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b38 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502348:0:2380:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bec00/0x36f9bdc9e421c244 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x278:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b38 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502355:0:2380:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bec00/0x36f9bdc9e421c244 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x278:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b38 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502362:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bec00/0x36f9bdc9e421c244 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x278:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b38 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502368:0:2380:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bec00/0x36f9bdc9e421c244 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x278:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b38 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764770.502376:0:2380:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970c03b80 x1862022966910592/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 3408/3368 e 0 to 0 dl 1775764786 ref 3 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000080:00010000:0.0:1775764770.502384:0:2380:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c244 from it ffff8f9970e08588 00010000:00010000:0.0:1775764770.502387:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bec00/0x36f9bdc9e421c244 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x278:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b38 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502393:0:2380:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bec00/0x36f9bdc9e421c244 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x278:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b38 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.502406:0:2380:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c24b lrc: 4/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x3/0x0 rrc: 8 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b54 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502415:0:2380:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c24b lrc: 4/1,0 mode: --/PR res: [0x200000401:0x287:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b54 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502426:0:2380:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c24b lrc: 4/1,0 mode: --/PR res: [0x200000401:0x287:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b54 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502433:0:2380:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c24b lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x287:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b54 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502440:0:2380:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c24b lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x287:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b54 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502447:0:2380:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c24b lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x287:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b54 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502453:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c24b lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x287:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b54 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502459:0:2380:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c24b lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x287:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b54 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764770.502468:0:2380:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970c03b80 x1862022966910592/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 3408/3368 e 0 to 0 dl 1775764786 ref 4 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000080:00010000:0.0:1775764770.502476:0:2380:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c24b from it ffff8f9970e09588 00010000:00010000:0.0:1775764770.502479:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c24b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x287:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b54 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502486:0:2380:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c24b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x287:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b54 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.502503:0:2380:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480400/0x36f9bdc9e421c252 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x3/0x0 rrc: 7 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b70 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502513:0:2380:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480400/0x36f9bdc9e421c252 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b70 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502519:0:2380:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480400/0x36f9bdc9e421c252 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b70 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502526:0:2380:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480400/0x36f9bdc9e421c252 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b70 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502534:0:2380:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480400/0x36f9bdc9e421c252 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b70 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502633:0:2380:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480400/0x36f9bdc9e421c252 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b70 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502641:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480400/0x36f9bdc9e421c252 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b70 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502648:0:2380:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480400/0x36f9bdc9e421c252 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b70 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764770.502657:0:2380:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970c03b80 x1862022966910592/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 3408/3368 e 0 to 0 dl 1775764786 ref 5 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000080:00010000:0.0:1775764770.502671:0:2380:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c252 from it ffff8f9970e0bd88 00010000:00010000:0.0:1775764770.502674:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480400/0x36f9bdc9e421c252 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b70 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502681:0:2380:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480400/0x36f9bdc9e421c252 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b70 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.502694:0:2380:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941481c00/0x36f9bdc9e421c259 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x3/0x0 rrc: 6 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b8c expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502703:0:2380:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941481c00/0x36f9bdc9e421c259 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b8c expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502709:0:2380:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941481c00/0x36f9bdc9e421c259 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b8c expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502716:0:2380:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941481c00/0x36f9bdc9e421c259 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b8c expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502723:0:2380:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941481c00/0x36f9bdc9e421c259 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b8c expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502729:0:2380:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941481c00/0x36f9bdc9e421c259 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b8c expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502735:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941481c00/0x36f9bdc9e421c259 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b8c expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502779:0:2380:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941481c00/0x36f9bdc9e421c259 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b8c expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764770.502788:0:2380:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970c03b80 x1862022966910592/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 3408/3368 e 0 to 0 dl 1775764786 ref 6 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000080:00010000:0.0:1775764770.502794:0:2380:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c259 from it ffff8f9970e0b188 00010000:00010000:0.0:1775764770.502797:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941481c00/0x36f9bdc9e421c259 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b8c expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502803:0:2380:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941481c00/0x36f9bdc9e421c259 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b8c expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.502816:0:2380:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c260 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x3/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8ba8 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502825:0:2380:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c260 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x292:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8ba8 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502831:0:2380:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c260 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8ba8 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502838:0:2380:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c260 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8ba8 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502845:0:2380:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c260 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8ba8 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502856:0:2380:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c260 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8ba8 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502862:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c260 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8ba8 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502868:0:2380:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c260 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8ba8 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764770.502875:0:2380:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970c03b80 x1862022966910592/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 3408/3368 e 0 to 0 dl 1775764786 ref 7 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000080:00010000:0.0:1775764770.502883:0:2380:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c260 from it ffff8f9970e0b588 00010000:00010000:0.0:1775764770.502885:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c260 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8ba8 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502892:0:2380:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c260 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8ba8 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.502904:0:2380:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5c00/0x36f9bdc9e421c267 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8bc4 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502913:0:2380:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5c00/0x36f9bdc9e421c267 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8bc4 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502919:0:2380:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5c00/0x36f9bdc9e421c267 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8bc4 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502933:0:2380:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5c00/0x36f9bdc9e421c267 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8bc4 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502941:0:2380:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5c00/0x36f9bdc9e421c267 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8bc4 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502948:0:2380:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5c00/0x36f9bdc9e421c267 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8bc4 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502954:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5c00/0x36f9bdc9e421c267 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8bc4 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502961:0:2380:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5c00/0x36f9bdc9e421c267 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8bc4 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764770.502970:0:2380:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970c03b80 x1862022966910592/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 3408/3368 e 0 to 0 dl 1775764786 ref 8 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000080:00010000:0.0:1775764770.502978:0:2380:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c267 from it ffff8f9970e08d88 00010000:00010000:0.0:1775764770.502981:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5c00/0x36f9bdc9e421c267 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8bc4 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.502988:0:2380:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5c00/0x36f9bdc9e421c267 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8bc4 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1775764770.503004:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970c03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966910592:192.168.202.146@tcp:63:ll_sa.0 00010000:00010000:0.0:1775764770.503040:0:131:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be200/0x36f9bdc9e421c236 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b00 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764770.503058:0:131:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x2b8:0x0], updating i_size 8192 00000080:00010000:0.0:1775764770.503063:0:131:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2b8:0x0](ffff8f9952d880c8) for lock 0x36f9bdc9e421c236 00000080:00010000:0.0:1775764770.503068:0:131:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c236 from it ffff8f9970e0a188 00010000:00010000:0.0:1775764770.503070:0:131:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be200/0x36f9bdc9e421c236 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b00 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.503077:0:131:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be200/0x36f9bdc9e421c236 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b00 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764770.503093:0:131:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf000/0x36f9bdc9e421c23d lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x281:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b1c expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764770.503102:0:131:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x281:0x0], updating i_size 7168 00000080:00010000:0.0:1775764770.503105:0:131:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x281:0x0](ffff8f9952dbd348) for lock 0x36f9bdc9e421c23d 00000080:00010000:0.0:1775764770.503108:0:131:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c23d from it ffff8f9970e09988 00010000:00010000:0.0:1775764770.503110:0:131:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf000/0x36f9bdc9e421c23d lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x281:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b1c expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.503116:0:131:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf000/0x36f9bdc9e421c23d lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x281:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b1c expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764770.503129:0:131:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bec00/0x36f9bdc9e421c244 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x278:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b38 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764770.503137:0:131:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x278:0x0], updating i_size 7168 00000080:00010000:0.0:1775764770.503140:0:131:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x278:0x0](ffff8f9952dbf448) for lock 0x36f9bdc9e421c244 00000080:00010000:0.0:1775764770.503143:0:131:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c244 from it ffff8f9970e08588 00010000:00010000:0.0:1775764770.503146:0:131:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bec00/0x36f9bdc9e421c244 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x278:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b38 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.503152:0:131:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bec00/0x36f9bdc9e421c244 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x278:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b38 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764770.503163:0:131:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c24b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x287:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b54 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764770.503170:0:131:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x287:0x0], updating i_size 7168 00000080:00010000:0.0:1775764770.503173:0:131:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x287:0x0](ffff8f9952dba1c8) for lock 0x36f9bdc9e421c24b 00000080:00010000:0.0:1775764770.503175:0:131:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c24b from it ffff8f9970e09588 00010000:00010000:0.0:1775764770.503178:0:131:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c24b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x287:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b54 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.503185:0:131:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c24b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x287:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b54 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764770.503198:0:131:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480400/0x36f9bdc9e421c252 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b70 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764770.503206:0:131:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x2a9:0x0], updating i_size 7168 00000080:00010000:0.0:1775764770.503209:0:131:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2a9:0x0](ffff8f9952da6c08) for lock 0x36f9bdc9e421c252 00000080:00010000:0.0:1775764770.503212:0:131:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c252 from it ffff8f9970e0bd88 00010000:00010000:0.0:1775764770.503215:0:131:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480400/0x36f9bdc9e421c252 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b70 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.503225:0:131:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480400/0x36f9bdc9e421c252 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b70 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764770.503239:0:131:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941481c00/0x36f9bdc9e421c259 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b8c expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764770.503555:0:131:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x2b1:0x0], updating i_size 7168 00000080:00010000:0.0:1775764770.503561:0:131:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2b1:0x0](ffff8f9952d142c8) for lock 0x36f9bdc9e421c259 00000080:00010000:0.0:1775764770.503564:0:131:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c259 from it ffff8f9970e0b188 00010000:00010000:0.0:1775764770.503567:0:131:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941481c00/0x36f9bdc9e421c259 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b8c expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.503575:0:131:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941481c00/0x36f9bdc9e421c259 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b8c expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764770.503590:0:131:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c260 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8ba8 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764770.503599:0:131:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x292:0x0], updating i_size 7168 00000080:00010000:0.0:1775764770.503612:0:131:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x292:0x0](ffff8f9952d89148) for lock 0x36f9bdc9e421c260 00000080:00010000:0.0:1775764770.503616:0:131:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c260 from it ffff8f9970e0b588 00010000:00010000:0.0:1775764770.503619:0:131:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c260 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8ba8 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.503633:0:131:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c260 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8ba8 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764770.503653:0:131:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5c00/0x36f9bdc9e421c267 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8bc4 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764770.503663:0:131:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x2e1:0x0], updating i_size 7168 00000080:00010000:0.0:1775764770.503666:0:131:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2e1:0x0](ffff8f9952d91148) for lock 0x36f9bdc9e421c267 00000080:00010000:0.0:1775764770.503669:0:131:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c267 from it ffff8f9970e08d88 00010000:00010000:0.0:1775764770.503671:0:131:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5c00/0x36f9bdc9e421c267 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8bc4 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.503677:0:131:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5c00/0x36f9bdc9e421c267 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8bc4 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000100:00100000:1.0:1775764770.509013:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764770.509046:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978357888 00000100:00100000:1.0:1775764770.509055:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978357888 00000100:00100000:1.0:1775764770.509103:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24101 00000100:00100000:1.0:1775764770.509124:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970c00a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+5:11866:x1862022978357888:12345-192.168.202.146@tcp:105: 00010000:00010000:1.0:1775764770.509141:0:5562:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0ca00/0x36f9bdc9e421c26e lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8bd9 expref: -99 pid: 19855 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.509154:0:5562:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0ca00/0x36f9bdc9e421c26e lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8bd9 expref: -99 pid: 19855 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.509162:0:5562:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0ca00/0x36f9bdc9e421c26e lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8bd9 expref: -99 pid: 19855 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.509171:0:5562:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0ca00/0x36f9bdc9e421c26e lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x10080000000000 nid: local remote: 0x2aac6a50e5f8bd9 expref: -99 pid: 19855 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.509180:0:5562:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0ca00/0x36f9bdc9e421c26e lrc: 5/1,0 mode: PR/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x10080000000000 nid: local remote: 0x2aac6a50e5f8bd9 expref: -99 pid: 19855 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.509188:0:5562:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0ca00/0x36f9bdc9e421c26e lrc: 5/1,0 mode: PR/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x80000000000 nid: local remote: 0x2aac6a50e5f8bd9 expref: -99 pid: 19855 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.509205:0:5562:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8f994ed0ca00) 00000100:00100000:1.0:1775764770.509245:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970c00a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+5:11866:x1862022978357888:12345-192.168.202.146@tcp:105: Request processed in 133us (243us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764770.509253:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24101 00000080:00200000:0.0:1775764770.532619:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2e6:0x0](ffff8f9952d921c8),name=ACCESS 00000002:00010000:0.0:1775764770.532639:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2e6:0x0]) in obj [0x200000401:0x26c:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764770.532647:0:19723:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/742 (0 0) 00010000:00010000:0.0:1775764770.532680:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c275 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.532687:0:19723:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c275 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.532696:0:19723:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c275 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764770.532712:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966911872:192.168.202.146@tcp:101:dbench.0 00000100:00100000:0.0:1775764770.533174:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16fb40 going to sleep for 11 seconds 00000080:00200000:0.0:1775764770.535887:0:19724:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2ec:0x0](ffff8f9952d63a88) 00000008:00010000:0.0:1775764770.536019:0:19724:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 1867776 blocks 4103 [cma]time 1775764748 1775764748 0 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000000c000ce/0x36f9bdc9e41cb1d1 lrc: 1/0,0 mode: PW/PW res: [0x280000403:0x6a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef8d6 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764770.536115:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.536121:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.536125:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:0.0:1775764770.536130:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.536132:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:0.0:1775764770.536134:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.536136:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:0.0:1775764770.536138:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.536149:0:19724:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=SEED, dir=[0x200000401:0x26b:0x0](ffff8f9952db00c8), flags=17 00000080:00200000:0.0:1775764770.536152:0:19724:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=SEED, dir=[0x200000401:0x26b:0x0](ffff8f9952db00c8), intent=0 00000002:00010000:0.0:1775764770.536168:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: SEED,[0x0:0x0:0x0]) in obj [0x200000401:0x26b:0x0], intent: lookup flags 0 00010000:00010000:0.0:1775764770.536209:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c283 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.536215:0:19724:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c283 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.536261:0:19724:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c283 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764770.536281:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966912128:192.168.202.146@tcp:101:dbench.0 00000100:00100000:0.0:1775764770.536403:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f600 going to sleep for 11 seconds 00010000:00010000:0.0:1775764770.536467:0:19857:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/657 (0 0) 00010000:00010000:0.0:1775764770.536490:0:19857:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421c28a lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 19857 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.536499:0:19857:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421c28a lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x3/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19857 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.536511:0:19857:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/682 (0 0) 00010000:00010000:0.0:1775764770.536516:0:19857:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c291 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x0/0x0 rrc: 6 type: IBT flags: 0x10000000000000 pid: 19857 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.536520:0:19857:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c291 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x3/0x0 rrc: 6 type: IBT flags: 0x0 pid: 19857 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.536528:0:19857:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/639 (0 0) 00010000:00010000:0.0:1775764770.536532:0:19857:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c298 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x0/0x0 rrc: 7 type: IBT flags: 0x10000000000000 pid: 19857 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.536536:0:19857:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c298 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x3/0x0 rrc: 7 type: IBT flags: 0x0 pid: 19857 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.536545:0:19857:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/690 (0 0) 00010000:00010000:0.0:1775764770.536549:0:19857:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c29f lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x0/0x0 rrc: 8 type: IBT flags: 0x10000000000000 pid: 19857 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.536552:0:19857:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c29f lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x3/0x0 rrc: 8 type: IBT flags: 0x0 pid: 19857 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.536561:0:19857:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/631 (0 0) 00010000:00010000:0.0:1775764770.536566:0:19857:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5c00/0x36f9bdc9e421c2a6 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x0/0x0 rrc: 9 type: IBT flags: 0x10000000000000 pid: 19857 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.536569:0:19857:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5c00/0x36f9bdc9e421c2a6 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x3/0x0 rrc: 9 type: IBT flags: 0x0 pid: 19857 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.536580:0:19857:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/645 (0 0) 00010000:00010000:0.0:1775764770.536585:0:19857:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c2ad lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x0/0x0 rrc: 10 type: IBT flags: 0x10000000000000 pid: 19857 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.536589:0:19857:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c2ad lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x3/0x0 rrc: 10 type: IBT flags: 0x0 pid: 19857 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.536598:0:19857:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49000/0x36f9bdc9e41cf8e2 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2ea:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4fd7be expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:0.0:1775764770.536605:0:19857:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49000/0x36f9bdc9e41cf8e2 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2ea:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4fd7be expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00010000:0.0:1775764770.536612:0:19857:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e41cf8e2 from it ffffafbac1ed7d48 00010000:00010000:0.0:1775764770.536617:0:19857:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49000/0x36f9bdc9e41cf8e2 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2ea:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e4fd7be expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:0.0:1775764770.536634:0:19857:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49000/0x36f9bdc9e41cf8e2 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ea:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4fd7be expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:0.0:1775764770.536659:0:19857:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/699 (0 0) 00010000:00010000:0.0:1775764770.536671:0:19857:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49400/0x36f9bdc9e421c2b4 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x0/0x0 rrc: 11 type: IBT flags: 0x10000000000000 pid: 19857 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.536677:0:19857:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49400/0x36f9bdc9e421c2b4 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x3/0x0 rrc: 11 type: IBT flags: 0x0 pid: 19857 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764770.536726:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966912256:192.168.202.146@tcp:63:ll_sa.0 00010000:00010000:1.0:1775764770.543218:0:2379:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421c28a lrc: 4/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x3/0x0 rrc: 11 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c3b expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543241:0:2379:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421c28a lrc: 4/1,0 mode: --/PR res: [0x200000401:0x291:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c3b expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543249:0:2379:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421c28a lrc: 4/1,0 mode: --/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c3b expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543261:0:2379:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421c28a lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8c3b expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543271:0:2379:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421c28a lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c3b expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543282:0:2379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421c28a lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8c3b expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543291:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421c28a lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8c3b expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543298:0:2379:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421c28a lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8c3b expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764770.543313:0:2379:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994f2e4380 x1862022966912256/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 3016/2976 e 0 to 0 dl 1775764786 ref 1 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000080:00010000:1.0:1775764770.543326:0:2379:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c28a from it ffff8f9966930988 00010000:00010000:1.0:1775764770.543330:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421c28a lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8c3b expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543338:0:2379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421c28a lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c3b expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.543370:0:2379:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c291 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x3/0x0 rrc: 10 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c57 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543383:0:2379:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c291 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2aa:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c57 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543389:0:2379:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c291 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2aa:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c57 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543398:0:2379:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c291 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2aa:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8c57 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543405:0:2379:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c291 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2aa:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c57 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543413:0:2379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c291 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2aa:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8c57 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543421:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c291 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2aa:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8c57 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543428:0:2379:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c291 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2aa:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8c57 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764770.543437:0:2379:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994f2e4380 x1862022966912256/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 3016/2976 e 0 to 0 dl 1775764786 ref 2 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000080:00010000:1.0:1775764770.543446:0:2379:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c291 from it ffff8f9966930188 00010000:00010000:1.0:1775764770.543448:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c291 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2aa:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8c57 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543457:0:2379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c291 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2aa:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c57 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.543467:0:2379:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c298 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x3/0x0 rrc: 9 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c73 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543473:0:2379:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c298 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c73 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543477:0:2379:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c298 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c73 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543482:0:2379:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c298 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8c73 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543486:0:2379:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c298 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c73 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543490:0:2379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c298 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8c73 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543494:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c298 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8c73 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543499:0:2379:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c298 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8c73 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764770.543508:0:2379:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994f2e4380 x1862022966912256/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 3016/2976 e 0 to 0 dl 1775764786 ref 3 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000080:00010000:1.0:1775764770.543512:0:2379:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c298 from it ffff8f9966933588 00010000:00010000:1.0:1775764770.543514:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c298 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8c73 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543518:0:2379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c298 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c73 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.543529:0:2379:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c29f lrc: 4/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x3/0x0 rrc: 8 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c8f expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543537:0:2379:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c29f lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c8f expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543544:0:2379:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c29f lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c8f expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543551:0:2379:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c29f lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8c8f expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543558:0:2379:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c29f lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c8f expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543565:0:2379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c29f lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8c8f expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543577:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c29f lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8c8f expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543583:0:2379:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c29f lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8c8f expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764770.543592:0:2379:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994f2e4380 x1862022966912256/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 3016/2976 e 0 to 0 dl 1775764786 ref 4 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000080:00010000:1.0:1775764770.543600:0:2379:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c29f from it ffff8f9966930d88 00010000:00010000:1.0:1775764770.543603:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c29f lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8c8f expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543617:0:2379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c29f lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c8f expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.543640:0:2379:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5c00/0x36f9bdc9e421c2a6 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x3/0x0 rrc: 7 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8cab expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543648:0:2379:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5c00/0x36f9bdc9e421c2a6 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x277:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8cab expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764770.543649:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966912128:192.168.202.146@tcp:101:dbench.0 00010000:00010000:1.0:1775764770.543655:0:2379:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5c00/0x36f9bdc9e421c2a6 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x277:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8cab expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543662:0:2379:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5c00/0x36f9bdc9e421c2a6 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x277:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8cab expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.543664:0:19724:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c283 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8d0d expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.543673:0:19724:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c283 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x1/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8d0d expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543674:0:2379:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5c00/0x36f9bdc9e421c2a6 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x277:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8cab expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.543680:0:19724:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c283 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x11/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8d0d expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543682:0:2379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5c00/0x36f9bdc9e421c2a6 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x277:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8cab expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.543688:0:19724:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c283 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x11/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8d0d expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543689:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5c00/0x36f9bdc9e421c2a6 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x277:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8cab expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.543695:0:19724:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c283 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x11/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8d0d expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543696:0:2379:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5c00/0x36f9bdc9e421c2a6 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x277:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8cab expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.543705:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c283 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x11/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8d0d expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764770.543705:0:2379:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994f2e4380 x1862022966912256/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 3016/2976 e 0 to 0 dl 1775764786 ref 5 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00010000:00010000:0.0:1775764770.543713:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c283 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x11/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8d0d expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:1.0:1775764770.543714:0:2379:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c2a6 from it ffff8f9966930588 00010000:00010000:1.0:1775764770.543716:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5c00/0x36f9bdc9e421c2a6 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x277:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8cab expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.543720:0:19724:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c283 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x11/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8d0d expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543724:0:2379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5c00/0x36f9bdc9e421c2a6 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x277:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8cab expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000002:00100000:0.0:1775764770.543728:0:19724:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=b, status=0 req@ffff8f994f2e6d80 x1862022966912128/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/584 e 0 to 0 dl 1775764786 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'dbench.0' uid:0 gid:0 projid:0 00010000:00010000:1.0:1775764770.543735:0:2379:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c2ad lrc: 4/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x3/0x0 rrc: 6 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8cc7 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:0.0:1775764770.543762:0:19724:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c283 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x11/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8d0d expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.543774:0:19724:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/741 (0 0) 00010000:00010000:1.0:1775764770.543774:0:2379:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c2ad lrc: 4/1,0 mode: --/PR res: [0x200000401:0x285:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8cc7 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1775764770.543781:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x26b:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1775764770.543785:0:2379:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c2ad lrc: 4/1,0 mode: --/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8cc7 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543791:0:2379:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c2ad lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8cc7 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543798:0:2379:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c2ad lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8cc7 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1775764770.543801:0:19724:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x2e5:0x0], updating i_size 7168 00000080:00200000:0.0:1775764770.543805:0:19724:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x2e5:0x0](ffff8f9952dd2a08): rc = 0 00010000:00010000:1.0:1775764770.543805:0:2379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c2ad lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8cc7 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1775764770.543810:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2e5:0x0](ffff8f9952dd2a08) for lock 0x36f9bdc9e421c283 00010000:00010000:1.0:1775764770.543811:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c2ad lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8cc7 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543816:0:2379:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c2ad lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8cc7 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764770.543823:0:2379:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994f2e4380 x1862022966912256/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 3016/2976 e 0 to 0 dl 1775764786 ref 6 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000080:00010000:0.0:1775764770.543827:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2e5:0x0](ffff8f9952dd2a08) 00000080:00010000:0.0:1775764770.543830:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c283 from it ffffafbac1c5ba28 00000080:00010000:1.0:1775764770.543831:0:2379:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c2ad from it ffff8f9966932588 00010000:00010000:1.0:1775764770.543833:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c2ad lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8cc7 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.543834:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c283 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x11/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8d0d expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.543841:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c283 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x11/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8d0d expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.543842:0:2379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c2ad lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8cc7 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.543859:0:2379:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49400/0x36f9bdc9e421c2b4 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x3/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8ce3 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543867:0:2379:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49400/0x36f9bdc9e421c2b4 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8ce3 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543873:0:2379:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49400/0x36f9bdc9e421c2b4 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8ce3 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543880:0:2379:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49400/0x36f9bdc9e421c2b4 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8ce3 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543886:0:2379:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49400/0x36f9bdc9e421c2b4 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8ce3 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1775764770.543888:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2e5:0x0](ffff8f9952dd2a08), inode mode 41c0 mask 1 00010000:00010000:1.0:1775764770.543892:0:2379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49400/0x36f9bdc9e421c2b4 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8ce3 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543899:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49400/0x36f9bdc9e421c2b4 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8ce3 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543904:0:2379:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49400/0x36f9bdc9e421c2b4 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8ce3 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1775764770.543907:0:19724:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=SMALL.FIL, dir=[0x200000401:0x2e5:0x0](ffff8f9952dd2a08), flags=1 00000080:00200000:0.0:1775764770.543911:0:19724:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=SMALL.FIL, dir=[0x200000401:0x2e5:0x0](ffff8f9952dd2a08), intent=getattr 00000002:00100000:1.0:1775764770.543912:0:2379:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994f2e4380 x1862022966912256/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 3016/2976 e 0 to 0 dl 1775764786 ref 7 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000080:00010000:1.0:1775764770.543924:0:2379:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c2b4 from it ffff8f9966931d88 00010000:00010000:1.0:1775764770.543927:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49400/0x36f9bdc9e421c2b4 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8ce3 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.543934:0:2379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49400/0x36f9bdc9e421c2b4 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8ce3 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000100:00100000:1.0:1775764770.543949:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966912256:192.168.202.146@tcp:63:ll_sa.0 00010000:00010000:1.0:1775764770.543978:0:21:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421c28a lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8c3b expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764770.543990:0:21:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x291:0x0], updating i_size 7168 00000080:00010000:1.0:1775764770.543993:0:21:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x291:0x0](ffff8f9952dccb08) for lock 0x36f9bdc9e421c28a 00000080:00010000:1.0:1775764770.543997:0:21:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c28a from it ffff8f9966930988 00010000:00010000:1.0:1775764770.544000:0:21:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421c28a lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8c3b expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.544007:0:21:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421c28a lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c3b expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764770.544024:0:21:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c291 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2aa:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8c57 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764770.544033:0:21:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x2aa:0x0], updating i_size 7168 00000080:00010000:1.0:1775764770.544035:0:21:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2aa:0x0](ffff8f9952d12a08) for lock 0x36f9bdc9e421c291 00000080:00010000:1.0:1775764770.544038:0:21:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c291 from it ffff8f9966930188 00010000:00010000:1.0:1775764770.544041:0:21:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c291 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2aa:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8c57 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.544047:0:21:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c291 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2aa:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c57 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764770.544059:0:21:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c298 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8c73 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764770.544066:0:21:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x27f:0x0], updating i_size 7168 00000080:00010000:1.0:1775764770.544069:0:21:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x27f:0x0](ffff8f9952db5348) for lock 0x36f9bdc9e421c298 00000080:00010000:1.0:1775764770.544071:0:21:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c298 from it ffff8f9966933588 00010000:00010000:1.0:1775764770.544074:0:21:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c298 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8c73 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.544080:0:21:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c298 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c73 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764770.544091:0:21:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c29f lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8c8f expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764770.544099:0:21:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x2b2:0x0], updating i_size 7168 00000080:00010000:1.0:1775764770.544101:0:21:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2b2:0x0](ffff8f9952d8ba88) for lock 0x36f9bdc9e421c29f 00000080:00010000:1.0:1775764770.544104:0:21:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c29f from it ffff8f9966930d88 00010000:00010000:1.0:1775764770.544107:0:21:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c29f lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8c8f expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.544113:0:21:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c29f lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c8f expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764770.544126:0:21:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5c00/0x36f9bdc9e421c2a6 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x277:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8cab expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764770.544134:0:21:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x277:0x0], updating i_size 7168 00000080:00010000:1.0:1775764770.544137:0:21:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x277:0x0](ffff8f9952e64b08) for lock 0x36f9bdc9e421c2a6 00000080:00010000:1.0:1775764770.544140:0:21:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c2a6 from it ffff8f9966930588 00010000:00010000:1.0:1775764770.544142:0:21:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5c00/0x36f9bdc9e421c2a6 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x277:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8cab expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.544149:0:21:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5c00/0x36f9bdc9e421c2a6 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x277:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8cab expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764770.544162:0:21:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c2ad lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8cc7 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764770.544171:0:21:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x285:0x0], updating i_size 7168 00000080:00010000:1.0:1775764770.544173:0:21:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x285:0x0](ffff8f9952da3a88) for lock 0x36f9bdc9e421c2ad 00000080:00010000:1.0:1775764770.544175:0:21:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c2ad from it ffff8f9966932588 00010000:00010000:1.0:1775764770.544177:0:21:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c2ad lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8cc7 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.544184:0:21:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c2ad lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8cc7 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764770.544196:0:21:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49400/0x36f9bdc9e421c2b4 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8ce3 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764770.544210:0:21:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x2bb:0x0], updating i_size 8192 00000080:00010000:1.0:1775764770.544212:0:21:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2bb:0x0](ffff8f9952da4b08) for lock 0x36f9bdc9e421c2b4 00000080:00010000:1.0:1775764770.544215:0:21:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c2b4 from it ffff8f9966931d88 00010000:00010000:1.0:1775764770.544218:0:21:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49400/0x36f9bdc9e421c2b4 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8ce3 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.544225:0:21:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49400/0x36f9bdc9e421c2b4 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8ce3 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000002:00010000:0.0:1775764770.544249:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: SMALL.FIL,[0x0:0x0:0x0]) in obj [0x200000401:0x2e5:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764770.544289:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd600/0x36f9bdc9e421c2bb lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2e5:0x0].0x0 bits 0x0/0x0 rrc: 6 type: IBT flags: 0x10000000000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.544297:0:19724:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd600/0x36f9bdc9e421c2bb lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2e5:0x0].0x0 bits 0x2/0x0 rrc: 6 type: IBT flags: 0x0 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.544306:0:19724:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd600/0x36f9bdc9e421c2bb lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2e5:0x0].0x0 bits 0x2/0x0 rrc: 6 type: IBT flags: 0x0 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764770.544330:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966912384:192.168.202.146@tcp:101:dbench.0 00000100:00100000:0.0:1775764770.544422:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9959502480 going to sleep for 11 seconds 00010000:00010000:1.0:1775764770.573655:0:11210:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764770.573663:0:11210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b9200/0x36f9bdc9e421c2c2 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f8d99 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.573672:0:11210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99668b9200 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:1.0:1775764770.573676:0:11210:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b9200/0x36f9bdc9e421c2c2 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e8:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f8d99 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.573684:0:11210:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b9200/0x36f9bdc9e421c2c2 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e8:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f8d99 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.573692:0:11210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b9200/0x36f9bdc9e421c2c2 lrc: 3/0,0 mode: PR/PR res: [0x200000401:0x2e8:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f8d99 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.573709:0:11210:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49a00/0x36f9bdc9e4217185 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e812a expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764770.573772:0:11210:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b9200/0x36f9bdc9e421c2c2 lrc: 2/0,0 mode: --/PR res: [0x200000401:0x2e8:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8d99 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.573780:0:11210:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764770.573805:0:11210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b9200/0x36f9bdc9e421c2c2 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2e8:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8d99 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.573810:0:11210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b9200/0x36f9bdc9e421c2c2 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2e8:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8d99 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.573819:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764770.573820:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764770.573821:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764770.573827:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1775764770.573839:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970c00380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966913408:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764770.585639:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970c00380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966913408:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764770.589250:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970d9f100 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966913664:192.168.202.146@tcp:36:dbench.0 00000080:00200000:0.0:1775764770.589303:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2e8:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764770.589314:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1775764770.589337:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:0.0:1775764770.589339:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:0.0:1775764770.589368:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2e8:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:1.0:1775764770.589698:0:5572:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764770.589706:0:5572:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0d200/0x36f9bdc9e421c2d0 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f8dd8 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.589714:0:5572:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f994ed0d200 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:1.0:1775764770.589718:0:5572:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0d200/0x36f9bdc9e421c2d0 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d3:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f8dd8 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.589727:0:5572:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0d200/0x36f9bdc9e421c2d0 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d3:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f8dd8 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.589735:0:5572:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0d200/0x36f9bdc9e421c2d0 lrc: 3/0,0 mode: PR/PR res: [0x200000401:0x2d3:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f8dd8 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.589937:0:5572:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d48200/0x36f9bdc9e421c014 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f84d2 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764770.590051:0:5572:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0d200/0x36f9bdc9e421c2d0 lrc: 2/0,0 mode: --/PR res: [0x200000401:0x2d3:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8dd8 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.590058:0:5572:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764770.590079:0:5572:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0d200/0x36f9bdc9e421c2d0 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2d3:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8dd8 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.590086:0:5572:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0d200/0x36f9bdc9e421c2d0 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2d3:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8dd8 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.590099:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764770.590101:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764770.590102:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764770.590110:0:2380:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1775764770.590125:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970c00380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966914048:192.168.202.146@tcp:103:ldlm_bl.0 00000080:00200000:0.0:1775764770.591537:0:19723:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2e8:0x0](ffff8f9952d93248) 00000008:00010000:0.0:1775764770.591596:0:19723:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 515 [cma]time 1775764730 1775764730 0 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001c16e60e/0x36f9bdc9e41cb15a lrc: 1/0,0 mode: PW/PW res: [0x240000403:0x69:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef78d expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 164 lru_type: 1 00000080:00200000:0.0:1775764770.593253:0:19723:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2e6:0x0](ffff8f9952d921c8) pos/size18446744073709551614/7168 32bit_api 0 00000080:00200008:0.0:1775764770.593277:0:19723:0:(file.c:447:ll_file_release()) START file ACCESS:[0x200000401:0x2e6:0x0](ffff8f9952d921c8), flags 304000 00010000:00010000:0.0:1775764770.593300:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e41cb137 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2e6:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ef716 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 92 lru_type: 1 00000080:00200000:0.0:1775764770.593338:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.593343:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.593349:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:0.0:1775764770.593354:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.593358:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:0.0:1775764770.593360:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.593363:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:0.0:1775764770.593365:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.593369:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=COREL, flags=1 00010000:00010000:0.0:1775764770.593381:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421c28a lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8c3b expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00010000:0.0:1775764770.593390:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c28a from it ffffafbac1c7baf8 00010000:00010000:0.0:1775764770.593393:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421c28a lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8c3b expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764770.593401:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421c28a lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c3b expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200008:0.0:1775764770.593435:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file COREL:[0x200000401:0x291:0x0](ffff8f9952dccb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764770.593440:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x291:0x0](ffff8f9952dccb08),name=COREL 00000002:00010000:0.0:1775764770.593457:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x291:0x0]) in obj [0x200000401:0x26c:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764770.593466:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421c28a lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c3b expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764770.593473:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421c28a lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c3b expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1775764770.593480:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26c:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764770.593486:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x291:0x0](ffff8f9952dccb08) 00000080:00010000:0.0:1775764770.593489:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x291:0x0](ffff8f9952dccb08) for lock 0x36f9bdc9e421c28a 00000080:00010000:0.0:1775764770.593493:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c28a from it ffffafbac1c7bcb8 00010000:00010000:0.0:1775764770.593496:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421c28a lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8c3b expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764770.593502:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421c28a lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c3b expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:0.0:1775764770.593534:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.593538:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.593540:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=19 00000080:00200000:0.0:1775764770.593543:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.593545:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=19 00000080:00200000:0.0:1775764770.593547:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.593550:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=19 00000080:00200000:0.0:1775764770.593552:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.593555:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=COREL, flags=259 00000080:00200000:0.0:1775764770.593559:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x291:0x0](ffff8f9952dccb08), inode mode 41c0 mask 44 00000080:00200008:0.0:1775764770.593565:0:19723:0:(file.c:1057:ll_file_open()) START file COREL:[0x200000401:0x291:0x0](ffff8f9952dccb08), flags 304000 00000080:00200008:0.0:1775764770.593581:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file COREL:[0x200000401:0x291:0x0](ffff8f9952dccb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764770.593585:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x291:0x0](ffff8f9952dccb08),name=COREL 00000002:00010000:0.0:1775764770.593592:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x291:0x0]) in obj [0x200000401:0x26c:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764770.593598:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421c28a lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c3b expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764770.593688:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421c28a lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c3b expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:0.0:1775764770.593696:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26c:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764770.593701:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x291:0x0](ffff8f9952dccb08) 00000080:00010000:0.0:1775764770.593704:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x291:0x0](ffff8f9952dccb08) for lock 0x36f9bdc9e421c28a 00000080:00010000:0.0:1775764770.593707:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c28a from it ffffafbac1c7bcd8 00010000:00010000:0.0:1775764770.593710:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421c28a lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8c3b expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764770.593721:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421c28a lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c3b expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:0.0:1775764770.594785:0:19723:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x291:0x0](ffff8f9952dccb08) pos/size0/7168 32bit_api 0 00000002:00010000:0.0:1775764770.594801:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x291:0x0]) in obj [0x200000401:0x291:0x0], intent: readdir flags 0 00010000:00010000:0.0:1775764770.594811:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421c28a lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c3b expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:0.0:1775764770.594820:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421c28a lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c3b expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000100:00100000:0.0:1775764770.595430:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7100 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966914304:192.168.202.146@tcp:37:dbench.0 00000100:00100000:0.0:1775764770.595464:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9959502480 going to sleep for 11 seconds 00000080:00200000:0.0:1775764770.597382:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2d3:0x0](ffff8f9952e2a1c8),name=ENROLL.PX 00000002:00010000:0.0:1775764770.597407:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2d3:0x0]) in obj [0x200000401:0x2b8:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764770.597425:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d48200/0x36f9bdc9e421c014 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2d3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f84d2 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764770.597437:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d48200/0x36f9bdc9e421c014 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2d3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f84d2 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1775764770.597445:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2b8:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764770.597453:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2d3:0x0](ffff8f9952e2a1c8) 00000080:00010000:0.0:1775764770.597456:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2d3:0x0](ffff8f9952e2a1c8) for lock 0x36f9bdc9e421c014 00000080:00010000:0.0:1775764770.597462:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c014 from it ffffafbac1c5bcb8 00010000:00010000:0.0:1775764770.597466:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d48200/0x36f9bdc9e421c014 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2d3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f84d2 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764770.597475:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d48200/0x36f9bdc9e421c014 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f84d2 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:0.0:1775764770.597856:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2d3:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1775764770.597868:0:19724:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2d3:0x0] 00000008:00010000:0.0:1775764770.597896:0:19724:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1775764770.597899:0:19724:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0df730, osc lock ffff8f996686f430, flags 0x400001002 00010000:00010000:0.0:1775764770.597916:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000054126077/0x36f9bdc9e41cae6d lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x60:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eefec expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 38 lru_type: 1 00000100:00100000:1.0:1775764770.597955:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970c00380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966914048:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1775764770.598069:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000054126077/0x36f9bdc9e41cae6d lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x60:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eefec expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 38 lru_type: 1 00000020:00200000:0.0:1775764770.598082:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1775764770.598086:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000054126077/0x36f9bdc9e41cae6d lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x60:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4eefec expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 38 lru_type: 1 00010000:00010000:0.0:1775764770.598095:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000054126077/0x36f9bdc9e41cae6d lrc: 4/0,0 mode: PW/PW res: [0x240000403:0x60:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eefec expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 39 lru_type: 1 00000020:00200000:0.0:1775764770.598105:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1775764770.598114:0:19724:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2d3:0x0] updating i_size 260096 i_blocks 1 00000020:00010000:0.0:1775764770.598118:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_release():245 00000020:00010000:0.0:1775764770.598121:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.598124:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0df730 (ffff8f9966dc2128/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.598402:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764770.598407:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0df730 (ffff8f9966dc2368/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764770.598436:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2d3:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764770.598469:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.598475:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.598479:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:0.0:1775764770.598485:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.598488:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=16 00000080:00200000:0.0:1775764770.598491:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.598494:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:0.0:1775764770.598496:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.598499:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=16 00000080:00200000:0.0:1775764770.598501:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.598506:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=ENROLL.PX, flags=0 00000080:00200000:0.0:1775764770.598515:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 3 00000080:00200000:0.0:1775764770.598519:0:19724:0:(namei.c:2395:ll_unlink()) VFS Op:name=ENROLL.PX, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8) 00010000:00010000:0.0:1775764770.598559:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be200/0x36f9bdc9e421c236 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f8b00 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00010000:00010000:0.0:1775764770.598575:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e41caad1 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x5/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ee710 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764770.598584:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970abee00/0x36f9bdc9e41cab09 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x11/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ee795 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 79 lru_type: 1 00010000:00010000:0.0:1775764770.599112:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d48200/0x36f9bdc9e421c014 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f84d2 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764770.599251:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2d3:0x0]: losing layout lock 00010000:00010000:1.0:1775764770.599306:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d48200/0x36f9bdc9e421c014 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2d3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f84d2 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764770.599314:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be200/0x36f9bdc9e421c236 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8b00 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1775764770.599319:0:19724:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1775764770.599322:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d48200/0x36f9bdc9e421c014 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2d3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f84d2 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764770.599339:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be200/0x36f9bdc9e421c236 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8b00 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00000020:00100000:1.0:1775764770.599353:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1775764770.599363:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970c00380 x1862022966914560/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764770.599385:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970c00380 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966914560:192.168.202.146@tcp:36:dbench.0 00000100:00100000:1.0:1775764770.599496:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f997130da80 going to sleep for 11 seconds 00000080:00200000:1.0:1775764770.601984:0:19862:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xebd:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764770.602001:0:19862:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xebd:0x0] 00000008:00010000:1.0:1775764770.602038:0:19862:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:1.0:1775764770.602042:0:19862:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99705ee730, osc lock ffff8f995875db50, flags 0x400041012 00010000:00010000:1.0:1775764770.602070:0:19862:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000bb07259e/0x36f9bdc9e421a67c lrc: 3/0,1 mode: PW/PW res: [0x240000403:0x653:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f37af expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764770.602090:0:19862:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000bb07259e/0x36f9bdc9e421a67c lrc: 3/0,1 mode: PW/PW res: [0x240000403:0x653:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f37af expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764770.602116:0:19862:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000bb07259e/0x36f9bdc9e421a67c lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x653:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5f37af expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764770.602128:0:19862:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000bb07259e/0x36f9bdc9e421a67c lrc: 4/0,0 mode: PW/PW res: [0x240000403:0x653:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f37af expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00000020:00010000:1.0:1775764770.602155:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_release():245 00000020:00010000:1.0:1775764770.602160:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764770.602163:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705ee730 (ffff8f99705b8ea8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764770.602167:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764770.602169:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705ee730 (ffff8f99705b8128/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764770.602552:0:19862:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xebd:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:1.0:1775764770.611051:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7100 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966914304:192.168.202.146@tcp:37:dbench.0 00010000:00010000:1.0:1775764770.613727:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421c28a lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8c3b expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764770.613788:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421c28a lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c3b expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00000080:00200000:1.0:1775764770.613945:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.613950:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.613956:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:1.0:1775764770.613963:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.613967:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:1.0:1775764770.613970:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.613974:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:1.0:1775764770.613977:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.613980:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=COREL, flags=17 00000080:00200000:1.0:1775764770.613983:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x291:0x0](ffff8f9952dccb08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.613997:0:19723:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=CDRROLS.CFG, dir=[0x200000401:0x291:0x0](ffff8f9952dccb08), flags=1 00000080:00200000:1.0:1775764770.614001:0:19723:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=CDRROLS.CFG, dir=[0x200000401:0x291:0x0](ffff8f9952dccb08), intent=getattr 00000002:00010000:1.0:1775764770.614025:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: CDRROLS.CFG,[0x0:0x0:0x0]) in obj [0x200000401:0x291:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764770.614502:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4000/0x36f9bdc9e421c301 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x291:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.614513:0:19723:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4000/0x36f9bdc9e421c301 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x291:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.614524:0:19723:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4000/0x36f9bdc9e421c301 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x291:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764770.614554:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970c03b80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966914816:192.168.202.146@tcp:101:dbench.0 00000100:00100000:1.0:1775764770.614686:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f997130dd80 going to sleep for 11 seconds 00000080:00200000:0.0:1775764770.619722:0:19862:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2bf:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1775764770.619733:0:19862:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2bf:0x0] 00000008:00010000:0.0:1775764770.619802:0:19862:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:0.0:1775764770.619807:0:19862:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0df730, osc lock ffff8f996686f430, flags 0x400041012 00010000:00010000:0.0:1775764770.619826:0:19862:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000071573dcf/0x36f9bdc9e41cabbf lrc: 3/0,1 mode: PW/PW res: [0x240000403:0x56:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee97f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 21 lru_type: 0 00010000:00010000:0.0:1775764770.619841:0:19862:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000071573dcf/0x36f9bdc9e41cabbf lrc: 3/0,1 mode: PW/PW res: [0x240000403:0x56:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee97f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 21 lru_type: 0 00010000:00010000:0.0:1775764770.619850:0:19862:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000071573dcf/0x36f9bdc9e41cabbf lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x56:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ee97f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 21 lru_type: 0 00010000:00010000:0.0:1775764770.619858:0:19862:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000071573dcf/0x36f9bdc9e41cabbf lrc: 4/0,0 mode: PW/PW res: [0x240000403:0x56:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee97f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 22 lru_type: 0 00000020:00010000:0.0:1775764770.619871:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_release():245 00000020:00010000:0.0:1775764770.619874:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.619877:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0df730 (ffff8f9966dc2128/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.619881:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764770.619884:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0df730 (ffff8f9966dc2368/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764770.619903:0:19862:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2bf:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764770.619923:0:19862:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2bc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1775764770.619928:0:19862:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2bc:0x0] 00000008:00010000:0.0:1775764770.619940:0:19862:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:0.0:1775764770.619942:0:19862:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0df730, osc lock ffff8f996686f430, flags 0x400041012 00010000:00010000:0.0:1775764770.619952:0:19862:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000415212e9/0x36f9bdc9e41cab48 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x54:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee83d expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 32 lru_type: 1 00010000:00010000:0.0:1775764770.619962:0:19862:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000415212e9/0x36f9bdc9e41cab48 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x54:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee83d expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 32 lru_type: 1 00010000:00010000:0.0:1775764770.619970:0:19862:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000415212e9/0x36f9bdc9e41cab48 lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x54:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ee83d expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 32 lru_type: 1 00010000:00010000:0.0:1775764770.619978:0:19862:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000415212e9/0x36f9bdc9e41cab48 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x54:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee83d expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 33 lru_type: 1 00000020:00010000:0.0:1775764770.619988:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_release():245 00000020:00010000:0.0:1775764770.619990:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.619993:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0df730 (ffff8f9966dc2128/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.619995:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764770.619997:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0df730 (ffff8f9966dc2368/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764770.620009:0:19862:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2bc:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764770.620026:0:19862:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2c3:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1775764770.620029:0:19862:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2c3:0x0] 00000008:00010000:0.0:1775764770.620037:0:19862:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:0.0:1775764770.620038:0:19862:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0df730, osc lock ffff8f996686f430, flags 0x400041012 00010000:00010000:0.0:1775764770.620045:0:19862:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000073099ecc/0x36f9bdc9e41cac4b lrc: 3/0,1 mode: PW/PW res: [0x240000403:0x58:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eeacf expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 28 lru_type: 1 00010000:00010000:0.0:1775764770.620051:0:19862:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000073099ecc/0x36f9bdc9e41cac4b lrc: 3/0,1 mode: PW/PW res: [0x240000403:0x58:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eeacf expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 28 lru_type: 1 00010000:00010000:0.0:1775764770.620056:0:19862:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000073099ecc/0x36f9bdc9e41cac4b lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x58:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4eeacf expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 28 lru_type: 1 00010000:00010000:0.0:1775764770.620065:0:19862:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000073099ecc/0x36f9bdc9e41cac4b lrc: 4/0,0 mode: PW/PW res: [0x240000403:0x58:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eeacf expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 29 lru_type: 1 00000020:00010000:0.0:1775764770.620071:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_release():245 00000020:00010000:0.0:1775764770.620072:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.620073:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0df730 (ffff8f9966dc2128/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.620075:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764770.620076:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0df730 (ffff8f9966dc2368/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764770.620083:0:19862:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2c3:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764770.620092:0:19862:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2be:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1775764770.620094:0:19862:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2be:0x0] 00000008:00010000:0.0:1775764770.620102:0:19862:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:0.0:1775764770.620104:0:19862:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0df730, osc lock ffff8f996686f430, flags 0x400041012 00010000:00010000:0.0:1775764770.620112:0:19862:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000076c55670/0x36f9bdc9e41cab87 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x55:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee8ec expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 32 lru_type: 1 00010000:00010000:0.0:1775764770.620120:0:19862:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000076c55670/0x36f9bdc9e41cab87 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x55:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee8ec expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 32 lru_type: 1 00010000:00010000:0.0:1775764770.620128:0:19862:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000076c55670/0x36f9bdc9e41cab87 lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x55:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ee8ec expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 32 lru_type: 1 00010000:00010000:0.0:1775764770.620135:0:19862:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000076c55670/0x36f9bdc9e41cab87 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x55:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee8ec expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 33 lru_type: 1 00000020:00010000:0.0:1775764770.620148:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_release():245 00000020:00010000:0.0:1775764770.620150:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.620153:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0df730 (ffff8f9966dc2128/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.620155:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764770.620157:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0df730 (ffff8f9966dc2368/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764770.620168:0:19862:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2be:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764770.620185:0:19862:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2df:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1775764770.620188:0:19862:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2df:0x0] 00000008:00010000:0.0:1775764770.620198:0:19862:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:0.0:1775764770.620200:0:19862:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0df730, osc lock ffff8f996686f430, flags 0x400041012 00010000:00010000:0.0:1775764770.620207:0:19862:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001ba52963/0x36f9bdc9e41cb011 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x66:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef3dc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 22 lru_type: 0 00010000:00010000:0.0:1775764770.620215:0:19862:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001ba52963/0x36f9bdc9e41cb011 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x66:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef3dc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 22 lru_type: 0 00010000:00010000:0.0:1775764770.620222:0:19862:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001ba52963/0x36f9bdc9e41cb011 lrc: 4/0,1 mode: PW/PW res: [0x240000403:0x66:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ef3dc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 22 lru_type: 0 00010000:00010000:0.0:1775764770.620229:0:19862:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001ba52963/0x36f9bdc9e41cb011 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x66:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef3dc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 0 00000020:00010000:0.0:1775764770.620238:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_release():245 00000020:00010000:0.0:1775764770.620241:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.620243:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0df730 (ffff8f9966dc2128/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.620246:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764770.620247:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0df730 (ffff8f9966dc2368/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764770.620258:0:19862:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2df:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:1.0:1775764770.621857:0:19861:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4600/0x36f9bdc9e421c308 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 19861 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.621866:0:19861:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4600/0x36f9bdc9e421c308 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19861 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.621886:0:19861:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4a00/0x36f9bdc9e421c30f lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 19861 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.621892:0:19861:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4a00/0x36f9bdc9e421c30f lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x3/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19861 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.621908:0:19861:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4800/0x36f9bdc9e421c316 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x0/0x0 rrc: 6 type: IBT flags: 0x10000000000000 pid: 19861 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.621913:0:19861:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4800/0x36f9bdc9e421c316 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x3/0x0 rrc: 6 type: IBT flags: 0x0 pid: 19861 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.621926:0:19861:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5000/0x36f9bdc9e421c31d lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x0/0x0 rrc: 7 type: IBT flags: 0x10000000000000 pid: 19861 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.621936:0:19861:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5000/0x36f9bdc9e421c31d lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x3/0x0 rrc: 7 type: IBT flags: 0x0 pid: 19861 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.621963:0:19861:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4400/0x36f9bdc9e421c324 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x0/0x0 rrc: 8 type: IBT flags: 0x10000000000000 pid: 19861 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.621969:0:19861:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4400/0x36f9bdc9e421c324 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x3/0x0 rrc: 8 type: IBT flags: 0x0 pid: 19861 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.621987:0:19861:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941481000/0x36f9bdc9e421b9a0 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2ba:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f711b expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764770.621997:0:19861:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941481000/0x36f9bdc9e421b9a0 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2ba:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f711b expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00010000:1.0:1775764770.622005:0:19861:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421b9a0 from it ffffafbac1ef7d48 00010000:00010000:1.0:1775764770.622008:0:19861:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941481000/0x36f9bdc9e421b9a0 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2ba:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f711b expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764770.622014:0:19861:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941481000/0x36f9bdc9e421b9a0 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ba:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f711b expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764770.622050:0:19861:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5a00/0x36f9bdc9e421c32b lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x0/0x0 rrc: 9 type: IBT flags: 0x10000000000000 pid: 19861 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.622056:0:19861:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5a00/0x36f9bdc9e421c32b lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x3/0x0 rrc: 9 type: IBT flags: 0x0 pid: 19861 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.622071:0:19861:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4200/0x36f9bdc9e421c332 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x0/0x0 rrc: 10 type: IBT flags: 0x10000000000000 pid: 19861 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.622077:0:19861:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4200/0x36f9bdc9e421c332 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x3/0x0 rrc: 10 type: IBT flags: 0x0 pid: 19861 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.622092:0:19861:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5200/0x36f9bdc9e421c339 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x0/0x0 rrc: 11 type: IBT flags: 0x10000000000000 pid: 19861 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.622098:0:19861:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5200/0x36f9bdc9e421c339 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x3/0x0 rrc: 11 type: IBT flags: 0x0 pid: 19861 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764770.622156:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970c02d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966914944:192.168.202.146@tcp:63:ll_sa.0 00000080:00200000:1.0:1775764770.622414:0:19861:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2ba:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764770.622423:0:19861:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2ba:0x0] 00000008:00010000:1.0:1775764770.622447:0:19861:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:1.0:1775764770.622450:0:19861:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99705ee530, osc lock ffff8f995875db50, flags 0x400041012 00010000:00010000:1.0:1775764770.622464:0:19861:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000083f2c617/0x36f9bdc9e41caaf4 lrc: 3/0,1 mode: PW/PW res: [0x240000403:0x54:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee756 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 67 lru_type: 1 00010000:00010000:1.0:1775764770.622475:0:19861:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000083f2c617/0x36f9bdc9e41caaf4 lrc: 3/0,1 mode: PW/PW res: [0x240000403:0x54:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee756 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 67 lru_type: 1 00010000:00010000:1.0:1775764770.622484:0:19861:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000083f2c617/0x36f9bdc9e41caaf4 lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x54:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ee756 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 67 lru_type: 1 00010000:00010000:1.0:1775764770.622492:0:19861:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000083f2c617/0x36f9bdc9e41caaf4 lrc: 4/0,0 mode: PW/PW res: [0x240000403:0x54:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee756 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 68 lru_type: 1 00000020:00010000:1.0:1775764770.622505:0:19861:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_release():245 00000020:00010000:1.0:1775764770.622509:0:19861:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764770.622512:0:19861:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705ee530 (ffff8f99705b8ea8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764770.622515:0:19861:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764770.622518:0:19861:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705ee530 (ffff8f99705b8128/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764770.622545:0:19861:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2ba:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:0.0:1775764770.629301:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970c03b80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966914816:192.168.202.146@tcp:101:dbench.0 00010000:00010000:0.0:1775764770.629314:0:19723:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4000/0x36f9bdc9e421c301 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x291:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8f05 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.629325:0:19723:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4000/0x36f9bdc9e421c301 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x293:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8f05 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.629330:0:19723:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4000/0x36f9bdc9e421c301 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x293:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8f05 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.629339:0:19723:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4000/0x36f9bdc9e421c301 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x293:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8f05 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.629346:0:19723:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4000/0x36f9bdc9e421c301 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x293:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8f05 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.629354:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4000/0x36f9bdc9e421c301 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x293:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8f05 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.629358:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4000/0x36f9bdc9e421c301 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x293:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8f05 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.629364:0:19723:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4000/0x36f9bdc9e421c301 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x293:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8f05 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764770.629374:0:19723:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970c03b80 x1862022966914816/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/928 e 0 to 0 dl 1775764786 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'dbench.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1775764770.629382:0:19723:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4000/0x36f9bdc9e421c301 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x293:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8f05 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:0.0:1775764770.629512:0:19723:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4000/0x36f9bdc9e421c301 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x293:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8f05 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.629523:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970f6c400/0x36f9bdc9e4217d01 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x293:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5ea5c7 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.629535:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970f6c400/0x36f9bdc9e4217d01 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x293:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5ea5c7 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.629543:0:19723:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4000/0x36f9bdc9e421c301 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x293:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8f05 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.629549:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4000/0x36f9bdc9e421c301 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x293:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x2aac6a50e5f8f05 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.629557:0:19723:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4000/0x36f9bdc9e421c301 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x293:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x2aac6a50e5f8f05 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.629615:0:19723:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000080:00200000:0.0:1775764770.629633:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x291:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1775764770.629695:0:19723:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x293:0x0](ffff8f9952dc9988): rc = 0 00000080:00200000:0.0:1775764770.629712:0:19723:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x293:0x0]: layout version change: 3 -> 3 00000080:00010000:0.0:1775764770.629721:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x293:0x0](ffff8f9952dc9988) for lock 0x36f9bdc9e4217d01 00000080:00010000:0.0:1775764770.629735:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x293:0x0](ffff8f9952dc9988) 00000080:00010000:0.0:1775764770.629768:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e4217d01 from it ffffafbac1c7bb20 00010000:00010000:0.0:1775764770.629774:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970f6c400/0x36f9bdc9e4217d01 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x293:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5ea5c7 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.629788:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970f6c400/0x36f9bdc9e4217d01 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x293:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5ea5c7 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200008:0.0:1775764770.629836:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file CDRROLS.CFG:[0x200000401:0x293:0x0](ffff8f9952dc9988), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764770.632172:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x293:0x0](ffff8f9952dc9988),name=CDRROLS.CFG 00000002:00010000:0.0:1775764770.632188:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x293:0x0]) in obj [0x200000401:0x291:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764770.632200:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970f6c400/0x36f9bdc9e4217d01 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x293:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5ea5c7 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764770.632211:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970f6c400/0x36f9bdc9e4217d01 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x293:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5ea5c7 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764770.632216:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x291:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764770.632220:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x293:0x0](ffff8f9952dc9988) 00000080:00010000:0.0:1775764770.632222:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x293:0x0](ffff8f9952dc9988) for lock 0x36f9bdc9e4217d01 00000080:00010000:0.0:1775764770.632227:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e4217d01 from it ffffafbac1c7bcb8 00010000:00010000:0.0:1775764770.632229:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970f6c400/0x36f9bdc9e4217d01 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x293:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5ea5c7 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764770.632235:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970f6c400/0x36f9bdc9e4217d01 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x293:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5ea5c7 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1775764770.632278:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x293:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1775764770.632285:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x293:0x0] 00000008:00010000:0.0:1775764770.632304:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1775764770.632307:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0df730, osc lock ffff8f996686f430, flags 0x400001002 00010000:00010000:0.0:1775764770.632318:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a988520c/0x36f9bdc9e41ca598 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x43:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ed964 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 91 lru_type: 1 00010000:00010000:0.0:1775764770.632325:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a988520c/0x36f9bdc9e41ca598 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x43:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ed964 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 91 lru_type: 1 00000020:00200000:0.0:1775764770.632333:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1775764770.632336:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a988520c/0x36f9bdc9e41ca598 lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x43:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ed964 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 91 lru_type: 1 00010000:00010000:0.0:1775764770.632341:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a988520c/0x36f9bdc9e41ca598 lrc: 4/0,0 mode: PW/PW res: [0x240000403:0x43:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ed964 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 92 lru_type: 1 00000020:00200000:0.0:1775764770.632347:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1775764770.632354:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x293:0x0] updating i_size 260096 i_blocks 515 00000020:00010000:0.0:1775764770.632356:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_release():245 00000020:00010000:0.0:1775764770.632358:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.632360:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0df730 (ffff8f9966dc2128/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.632362:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764770.632364:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0df730 (ffff8f9966dc2368/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764770.632380:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x293:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764770.632420:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.632426:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.632430:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:0.0:1775764770.632435:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.632438:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=16 00000080:00200000:0.0:1775764770.632441:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.632444:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:0.0:1775764770.632446:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.632450:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=COREL, flags=16 00000080:00200000:0.0:1775764770.632452:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x291:0x0](ffff8f9952dccb08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.632457:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=CDRROLS.CFG, flags=0 00000080:00200000:0.0:1775764770.632464:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x291:0x0](ffff8f9952dccb08), inode mode 41c0 mask 3 00000080:00200000:0.0:1775764770.632469:0:19723:0:(namei.c:2395:ll_unlink()) VFS Op:name=CDRROLS.CFG, dir=[0x200000401:0x291:0x0](ffff8f9952dccb08) 00010000:00010000:0.0:1775764770.632505:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421c28a lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f8c3b expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00010000:00010000:0.0:1775764770.632521:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970fd7200/0x36f9bdc9e41ca567 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x291:0x0].0x0 bits 0x5/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ed8e6 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 91 lru_type: 1 00010000:00010000:0.0:1775764770.632532:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99700e4000/0x36f9bdc9e41ca59f lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x11/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ed980 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00010000:00010000:0.0:1775764770.632593:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4000/0x36f9bdc9e421c301 lrc: 3/0,0 mode: PR/PR res: [0x200000401:0x293:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f8f05 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1775764770.632615:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x293:0x0]: losing layout lock 00010000:00010000:0.0:1775764770.632641:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970f6c400/0x36f9bdc9e4217d01 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x293:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5ea5c7 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1775764770.632657:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x293:0x0]: losing layout lock 00010000:00010000:0.0:1775764770.632682:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4000/0x36f9bdc9e421c301 lrc: 2/0,0 mode: --/PR res: [0x200000401:0x293:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8f05 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.632690:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970f6c400/0x36f9bdc9e4217d01 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x293:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5ea5c7 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764770.632697:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421c28a lrc: 1/0,0 mode: --/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8c3b expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00010000:00010000:0.0:1775764770.632702:0:19723:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:0.0:1775764770.632705:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970f6c400/0x36f9bdc9e4217d01 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x293:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5ea5c7 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764770.632715:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421c28a lrc: 0/0,0 mode: --/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8c3b expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00000020:00100000:0.0:1775764770.632727:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 2 is allocated opc 36, max 7 00000100:00100000:0.0:1775764770.632735:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e4a80 x1862022966915840/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 504/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764770.632786:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966915840:192.168.202.146@tcp:36:dbench.0 00000100:00100000:0.0:1775764770.632837:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f540 going to sleep for 11 seconds 00010000:00010000:1.0:1775764770.635128:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764770.635142:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4000/0x36f9bdc9e421c301 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x293:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8f05 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635152:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970dc4000 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:1.0:1775764770.635156:0:19801:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4000/0x36f9bdc9e421c301 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x293:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8f05 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635164:0:19801:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4000/0x36f9bdc9e421c301 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x293:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8f05 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635171:0:19801:0:(ldlm_request.c:1765:ldlm_cli_cancel()) ### lock is already being destroyed 00010000:00010000:1.0:1775764770.635173:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4000/0x36f9bdc9e421c301 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x293:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8f05 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635180:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4000/0x36f9bdc9e421c301 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x293:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8f05 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635251:0:2380:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4600/0x36f9bdc9e421c308 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x3/0x0 rrc: 18 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8eb1 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635261:0:2380:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4600/0x36f9bdc9e421c308 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2c5:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8eb1 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635270:0:2380:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4600/0x36f9bdc9e421c308 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2c5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8eb1 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635278:0:2380:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4600/0x36f9bdc9e421c308 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2c5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8eb1 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635355:0:2380:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4600/0x36f9bdc9e421c308 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2c5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8eb1 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635367:0:2380:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4600/0x36f9bdc9e421c308 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2c5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8eb1 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635376:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4600/0x36f9bdc9e421c308 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2c5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8eb1 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635383:0:2380:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4600/0x36f9bdc9e421c308 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2c5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8eb1 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764770.635404:0:2380:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970c02d80 x1862022966914944/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 3472/6120 e 0 to 0 dl 1775764786 ref 1 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:1.0:1775764770.635428:0:2380:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4600/0x36f9bdc9e421c308 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8eb1 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:1.0:1775764770.635442:0:2380:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c308 from it ffff8f9970e0a588 00010000:00010000:1.0:1775764770.635446:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4600/0x36f9bdc9e421c308 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8eb1 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635459:0:2380:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4600/0x36f9bdc9e421c308 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8eb1 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.635490:0:2380:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4a00/0x36f9bdc9e421c30f lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x3/0x0 rrc: 17 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8ed4 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635501:0:2380:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4a00/0x36f9bdc9e421c30f lrc: 4/1,0 mode: --/PR res: [0x200000401:0xeba:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8ed4 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635508:0:2380:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4a00/0x36f9bdc9e421c30f lrc: 4/1,0 mode: --/PR res: [0x200000401:0xeba:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8ed4 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635515:0:2380:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4a00/0x36f9bdc9e421c30f lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xeba:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8ed4 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635523:0:2380:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4a00/0x36f9bdc9e421c30f lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xeba:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8ed4 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635530:0:2380:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4a00/0x36f9bdc9e421c30f lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xeba:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8ed4 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635538:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4a00/0x36f9bdc9e421c30f lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xeba:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8ed4 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635544:0:2380:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4a00/0x36f9bdc9e421c30f lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xeba:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8ed4 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764770.635558:0:2380:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970c02d80 x1862022966914944/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 3472/6120 e 0 to 0 dl 1775764786 ref 2 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:1.0:1775764770.635567:0:2380:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4a00/0x36f9bdc9e421c30f lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xeba:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8ed4 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:1.0:1775764770.635575:0:2380:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c30f from it ffff8f9970e0a188 00010000:00010000:1.0:1775764770.635578:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4a00/0x36f9bdc9e421c30f lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xeba:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8ed4 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635584:0:2380:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4a00/0x36f9bdc9e421c30f lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xeba:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8ed4 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.635596:0:2380:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4800/0x36f9bdc9e421c316 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x3/0x0 rrc: 16 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8efe expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635605:0:2380:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4800/0x36f9bdc9e421c316 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2c7:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8efe expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635611:0:2380:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4800/0x36f9bdc9e421c316 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2c7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8efe expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635619:0:2380:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4800/0x36f9bdc9e421c316 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2c7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8efe expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635637:0:2380:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4800/0x36f9bdc9e421c316 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2c7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8efe expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635650:0:2380:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4800/0x36f9bdc9e421c316 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2c7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8efe expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635657:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4800/0x36f9bdc9e421c316 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2c7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8efe expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635663:0:2380:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4800/0x36f9bdc9e421c316 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2c7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8efe expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764770.635672:0:2380:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970c02d80 x1862022966914944/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 3472/6120 e 0 to 0 dl 1775764786 ref 3 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:1.0:1775764770.635680:0:2380:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4800/0x36f9bdc9e421c316 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8efe expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:1.0:1775764770.635688:0:2380:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c316 from it ffff8f9970e08588 00010000:00010000:1.0:1775764770.635691:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4800/0x36f9bdc9e421c316 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8efe expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635698:0:2380:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4800/0x36f9bdc9e421c316 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8efe expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.635711:0:2380:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5000/0x36f9bdc9e421c31d lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x3/0x0 rrc: 15 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8f21 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635720:0:2380:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5000/0x36f9bdc9e421c31d lrc: 4/1,0 mode: --/PR res: [0x200000401:0xec8:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8f21 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635731:0:2380:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5000/0x36f9bdc9e421c31d lrc: 4/1,0 mode: --/PR res: [0x200000401:0xec8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8f21 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635776:0:2380:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5000/0x36f9bdc9e421c31d lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xec8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8f21 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635785:0:2380:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5000/0x36f9bdc9e421c31d lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xec8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8f21 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635793:0:2380:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5000/0x36f9bdc9e421c31d lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xec8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8f21 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635799:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5000/0x36f9bdc9e421c31d lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xec8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8f21 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635806:0:2380:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5000/0x36f9bdc9e421c31d lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xec8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8f21 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764770.635815:0:2380:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970c02d80 x1862022966914944/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 3472/6120 e 0 to 0 dl 1775764786 ref 4 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:1.0:1775764770.635823:0:2380:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5000/0x36f9bdc9e421c31d lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xec8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8f21 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:1.0:1775764770.635832:0:2380:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c31d from it ffff8f9970e09588 00010000:00010000:1.0:1775764770.635834:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5000/0x36f9bdc9e421c31d lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xec8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8f21 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635842:0:2380:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5000/0x36f9bdc9e421c31d lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xec8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8f21 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.635852:0:2380:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4400/0x36f9bdc9e421c324 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x3/0x0 rrc: 14 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8f3d expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635858:0:2380:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4400/0x36f9bdc9e421c324 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2c1:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8f3d expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635862:0:2380:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4400/0x36f9bdc9e421c324 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2c1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8f3d expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635867:0:2380:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4400/0x36f9bdc9e421c324 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2c1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8f3d expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635871:0:2380:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4400/0x36f9bdc9e421c324 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2c1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8f3d expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635875:0:2380:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4400/0x36f9bdc9e421c324 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2c1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8f3d expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635878:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4400/0x36f9bdc9e421c324 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2c1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8f3d expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635882:0:2380:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4400/0x36f9bdc9e421c324 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2c1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8f3d expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764770.635891:0:2380:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970c02d80 x1862022966914944/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 3472/6120 e 0 to 0 dl 1775764786 ref 5 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:1.0:1775764770.635896:0:2380:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4400/0x36f9bdc9e421c324 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8f3d expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:1.0:1775764770.635903:0:2380:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c324 from it ffff8f9970e0bd88 00010000:00010000:1.0:1775764770.635904:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4400/0x36f9bdc9e421c324 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8f3d expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635908:0:2380:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4400/0x36f9bdc9e421c324 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8f3d expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.635915:0:2380:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5a00/0x36f9bdc9e421c32b lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x3/0x0 rrc: 13 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8f75 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635920:0:2380:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5a00/0x36f9bdc9e421c32b lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2d1:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8f75 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635925:0:2380:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5a00/0x36f9bdc9e421c32b lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2d1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8f75 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635932:0:2380:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5a00/0x36f9bdc9e421c32b lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2d1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8f75 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635939:0:2380:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5a00/0x36f9bdc9e421c32b lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2d1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8f75 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635951:0:2380:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5a00/0x36f9bdc9e421c32b lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2d1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8f75 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635957:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5a00/0x36f9bdc9e421c32b lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2d1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8f75 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635963:0:2380:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5a00/0x36f9bdc9e421c32b lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2d1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8f75 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764770.635971:0:2380:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970c02d80 x1862022966914944/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 3472/6120 e 0 to 0 dl 1775764786 ref 6 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:1.0:1775764770.635978:0:2380:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5a00/0x36f9bdc9e421c32b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2d1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8f75 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:1.0:1775764770.635987:0:2380:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c32b from it ffff8f9970e0b188 00010000:00010000:1.0:1775764770.635990:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5a00/0x36f9bdc9e421c32b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2d1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8f75 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.635997:0:2380:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5a00/0x36f9bdc9e421c32b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8f75 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.636007:0:2380:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4200/0x36f9bdc9e421c332 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x3/0x0 rrc: 12 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8f91 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.636012:0:2380:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4200/0x36f9bdc9e421c332 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2db:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8f91 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.636020:0:2380:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4200/0x36f9bdc9e421c332 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2db:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8f91 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.636024:0:2380:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4200/0x36f9bdc9e421c332 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2db:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8f91 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.636028:0:2380:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4200/0x36f9bdc9e421c332 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2db:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8f91 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.636032:0:2380:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4200/0x36f9bdc9e421c332 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2db:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8f91 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.636036:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4200/0x36f9bdc9e421c332 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2db:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8f91 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.636040:0:2380:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4200/0x36f9bdc9e421c332 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2db:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8f91 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764770.636045:0:2380:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970c02d80 x1862022966914944/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 3472/6120 e 0 to 0 dl 1775764786 ref 7 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:1.0:1775764770.636049:0:2380:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4200/0x36f9bdc9e421c332 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2db:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8f91 expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:1.0:1775764770.636054:0:2380:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c332 from it ffff8f9970e0b588 00010000:00010000:1.0:1775764770.636055:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4200/0x36f9bdc9e421c332 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2db:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8f91 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.636063:0:2380:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4200/0x36f9bdc9e421c332 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2db:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8f91 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.636070:0:2380:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5200/0x36f9bdc9e421c339 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x3/0x0 rrc: 11 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8fad expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.636074:0:2380:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5200/0x36f9bdc9e421c339 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2cd:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8fad expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.636079:0:2380:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5200/0x36f9bdc9e421c339 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2cd:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8fad expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.636084:0:2380:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5200/0x36f9bdc9e421c339 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2cd:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8fad expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.636087:0:2380:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5200/0x36f9bdc9e421c339 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2cd:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8fad expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.636091:0:2380:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5200/0x36f9bdc9e421c339 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2cd:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8fad expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.636095:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5200/0x36f9bdc9e421c339 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2cd:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8fad expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.636098:0:2380:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5200/0x36f9bdc9e421c339 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2cd:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8fad expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764770.636105:0:2380:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970c02d80 x1862022966914944/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 3472/6120 e 0 to 0 dl 1775764786 ref 8 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:1.0:1775764770.636110:0:2380:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5200/0x36f9bdc9e421c339 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2cd:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8fad expref: -99 pid: 19861 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:1.0:1775764770.636114:0:2380:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c339 from it ffff8f9970e08d88 00010000:00010000:1.0:1775764770.636116:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5200/0x36f9bdc9e421c339 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2cd:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8fad expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.636119:0:2380:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5200/0x36f9bdc9e421c339 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2cd:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8fad expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000100:00100000:1.0:1775764770.636130:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970c02d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966914944:192.168.202.146@tcp:63:ll_sa.0 00010000:00010000:1.0:1775764770.636163:0:21:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4600/0x36f9bdc9e421c308 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8eb1 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764770.636232:0:21:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x2c5:0x0](ffff8f9952d8ec08): rc = 0 00000080:00200000:1.0:1775764770.636245:0:21:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x2c5:0x0]: layout version change: 3 -> 3 00000080:00010000:1.0:1775764770.636252:0:21:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2c5:0x0](ffff8f9952d8ec08) for lock 0x36f9bdc9e421c308 00000080:00010000:1.0:1775764770.636266:0:21:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c308 from it ffff8f9970e0a588 00010000:00010000:1.0:1775764770.636268:0:21:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4600/0x36f9bdc9e421c308 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8eb1 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.636272:0:21:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4600/0x36f9bdc9e421c308 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8eb1 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764770.636288:0:21:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4a00/0x36f9bdc9e421c30f lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xeba:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8ed4 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764770.636308:0:21:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0xeba:0x0](ffff8f9952eedb88): rc = 0 00000080:00200000:1.0:1775764770.636316:0:21:0:(file.c:6766:ll_layout_conf()) [0x200000401:0xeba:0x0]: layout version change: 3 -> 3 00000080:00010000:1.0:1775764770.636321:0:21:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xeba:0x0](ffff8f9952eedb88) for lock 0x36f9bdc9e421c30f 00000080:00010000:1.0:1775764770.636323:0:21:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c30f from it ffff8f9970e0a188 00010000:00010000:1.0:1775764770.636325:0:21:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4a00/0x36f9bdc9e421c30f lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xeba:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8ed4 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.636331:0:21:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4a00/0x36f9bdc9e421c30f lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xeba:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8ed4 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764770.636344:0:21:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4800/0x36f9bdc9e421c316 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8efe expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764770.636373:0:21:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x2c7:0x0](ffff8f9952d97448): rc = 0 00000080:00200000:1.0:1775764770.636386:0:21:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x2c7:0x0]: layout version change: 3 -> 3 00000080:00010000:1.0:1775764770.636394:0:21:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2c7:0x0](ffff8f9952d97448) for lock 0x36f9bdc9e421c316 00000080:00010000:1.0:1775764770.636397:0:21:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c316 from it ffff8f9970e08588 00010000:00010000:1.0:1775764770.636400:0:21:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4800/0x36f9bdc9e421c316 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8efe expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.636407:0:21:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4800/0x36f9bdc9e421c316 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8efe expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764770.636425:0:21:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5000/0x36f9bdc9e421c31d lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xec8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8f21 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764770.636456:0:21:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0xec8:0x0](ffff8f9952eb42c8): rc = 0 00000080:00200000:1.0:1775764770.636472:0:21:0:(file.c:6766:ll_layout_conf()) [0x200000401:0xec8:0x0]: layout version change: 3 -> 3 00000080:00010000:1.0:1775764770.636480:0:21:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xec8:0x0](ffff8f9952eb42c8) for lock 0x36f9bdc9e421c31d 00000080:00010000:1.0:1775764770.636484:0:21:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c31d from it ffff8f9970e09588 00010000:00010000:1.0:1775764770.636488:0:21:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5000/0x36f9bdc9e421c31d lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xec8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8f21 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.636495:0:21:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5000/0x36f9bdc9e421c31d lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xec8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8f21 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764770.636509:0:21:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4400/0x36f9bdc9e421c324 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8f3d expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764770.636539:0:21:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x2c1:0x0](ffff8f9952e4d348): rc = 0 00000080:00200000:1.0:1775764770.636555:0:21:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x2c1:0x0]: layout version change: 3 -> 3 00000080:00010000:1.0:1775764770.636563:0:21:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2c1:0x0](ffff8f9952e4d348) for lock 0x36f9bdc9e421c324 00000080:00010000:1.0:1775764770.636567:0:21:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c324 from it ffff8f9970e0bd88 00010000:00010000:1.0:1775764770.636570:0:21:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4400/0x36f9bdc9e421c324 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8f3d expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.636578:0:21:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4400/0x36f9bdc9e421c324 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8f3d expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764770.636596:0:21:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5a00/0x36f9bdc9e421c32b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2d1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8f75 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764770.636639:0:21:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x2d1:0x0](ffff8f9952d942c8): rc = 0 00000080:00200000:1.0:1775764770.636657:0:21:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x2d1:0x0]: layout version change: 3 -> 3 00000080:00010000:1.0:1775764770.636666:0:21:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2d1:0x0](ffff8f9952d942c8) for lock 0x36f9bdc9e421c32b 00000080:00010000:1.0:1775764770.636671:0:21:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c32b from it ffff8f9970e0b188 00010000:00010000:1.0:1775764770.636674:0:21:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5a00/0x36f9bdc9e421c32b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2d1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8f75 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.636681:0:21:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5a00/0x36f9bdc9e421c32b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8f75 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764770.636695:0:21:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4200/0x36f9bdc9e421c332 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2db:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8f91 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764770.636728:0:21:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x2db:0x0](ffff8f9952dd3248): rc = 0 00000080:00200000:1.0:1775764770.636768:0:21:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x2db:0x0]: layout version change: 3 -> 3 00000080:00010000:1.0:1775764770.636778:0:21:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2db:0x0](ffff8f9952dd3248) for lock 0x36f9bdc9e421c332 00000080:00010000:1.0:1775764770.636782:0:21:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c332 from it ffff8f9970e0b588 00010000:00010000:1.0:1775764770.636785:0:21:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4200/0x36f9bdc9e421c332 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2db:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8f91 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.636792:0:21:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4200/0x36f9bdc9e421c332 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2db:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8f91 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764770.636812:0:21:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5200/0x36f9bdc9e421c339 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2cd:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8fad expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764770.636840:0:21:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x2cd:0x0](ffff8f9952d92a08): rc = 0 00000080:00200000:1.0:1775764770.636849:0:21:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x2cd:0x0]: layout version change: 3 -> 3 00000080:00010000:1.0:1775764770.636854:0:21:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2cd:0x0](ffff8f9952d92a08) for lock 0x36f9bdc9e421c339 00000080:00010000:1.0:1775764770.636857:0:21:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c339 from it ffff8f9970e08d88 00010000:00010000:1.0:1775764770.636858:0:21:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5200/0x36f9bdc9e421c339 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2cd:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8fad expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.636863:0:21:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5200/0x36f9bdc9e421c339 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2cd:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8fad expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764770.636899:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764770.636901:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764770.636903:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1775764770.637054:0:19862:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2c5:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1775764770.637062:0:19862:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2c5:0x0] 00000008:00010000:0.0:1775764770.637082:0:19862:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:0.0:1775764770.637085:0:19862:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0df730, osc lock ffff8f996686f430, flags 0x400041012 00010000:00010000:0.0:1775764770.637103:0:19862:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ac098a84/0x36f9bdc9e41cac83 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x59:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eeb54 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 0 00010000:00010000:0.0:1775764770.637114:0:19862:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ac098a84/0x36f9bdc9e41cac83 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x59:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eeb54 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 0 00010000:00010000:0.0:1775764770.637120:0:19862:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ac098a84/0x36f9bdc9e41cac83 lrc: 4/0,1 mode: PW/PW res: [0x240000403:0x59:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4eeb54 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 0 00010000:00010000:0.0:1775764770.637125:0:19862:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ac098a84/0x36f9bdc9e41cac83 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x59:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eeb54 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 0 00000020:00010000:0.0:1775764770.637133:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_release():245 00000020:00010000:0.0:1775764770.637135:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.637136:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0df730 (ffff8f9966dc2128/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.637139:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764770.637140:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0df730 (ffff8f9966dc2368/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764770.637155:0:19862:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2c5:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764770.637166:0:19862:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xeba:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1775764770.637169:0:19862:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xeba:0x0] 00000008:00010000:0.0:1775764770.637177:0:19862:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:0.0:1775764770.637178:0:19862:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0df730, osc lock ffff8f996686f430, flags 0x400041012 00010000:00010000:0.0:1775764770.637185:0:19862:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000839c629c/0x36f9bdc9e421a3c0 lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x651:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f2e5c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00010000:00010000:0.0:1775764770.637191:0:19862:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000839c629c/0x36f9bdc9e421a3c0 lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x651:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f2e5c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00010000:00010000:0.0:1775764770.637197:0:19862:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000839c629c/0x36f9bdc9e421a3c0 lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x651:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5f2e5c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00010000:00010000:0.0:1775764770.637206:0:19862:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000839c629c/0x36f9bdc9e421a3c0 lrc: 4/0,0 mode: PW/PW res: [0x280000403:0x651:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f2e5c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 0 00000020:00010000:0.0:1775764770.637213:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_release():245 00000020:00010000:0.0:1775764770.637214:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.637216:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0df730 (ffff8f9966dc2128/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.637218:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764770.637219:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0df730 (ffff8f9966dc2368/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764770.637225:0:19862:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xeba:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764770.637234:0:19862:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2c7:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1775764770.637236:0:19862:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2c7:0x0] 00000008:00010000:0.0:1775764770.637249:0:19862:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:0.0:1775764770.637251:0:19862:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0df730, osc lock ffff8f996686f430, flags 0x400041012 00010000:00010000:0.0:1775764770.637256:0:19862:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001fc9e7d4/0x36f9bdc9e41cacc9 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x5a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eebfc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 46 lru_type: 1 00010000:00010000:0.0:1775764770.637261:0:19862:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001fc9e7d4/0x36f9bdc9e41cacc9 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x5a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eebfc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 46 lru_type: 1 00010000:00010000:0.0:1775764770.637266:0:19862:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001fc9e7d4/0x36f9bdc9e41cacc9 lrc: 4/0,1 mode: PW/PW res: [0x240000403:0x5a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4eebfc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 46 lru_type: 1 00010000:00010000:0.0:1775764770.637271:0:19862:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001fc9e7d4/0x36f9bdc9e41cacc9 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x5a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eebfc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 47 lru_type: 1 00000020:00010000:0.0:1775764770.637281:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_release():245 00000020:00010000:0.0:1775764770.637282:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.637284:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0df730 (ffff8f9966dc2128/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.637286:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764770.637287:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0df730 (ffff8f9966dc2368/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764770.637294:0:19862:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2c7:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764770.637302:0:19862:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xec8:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1775764770.637305:0:19862:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xec8:0x0] 00000008:00010000:0.0:1775764770.637313:0:19862:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:0.0:1775764770.637315:0:19862:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0df730, osc lock ffff8f996686f430, flags 0x400041012 00010000:00010000:0.0:1775764770.637322:0:19862:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000030aeec1d/0x36f9bdc9e421ad75 lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x658:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f4c54 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 0 00010000:00010000:0.0:1775764770.637332:0:19862:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000030aeec1d/0x36f9bdc9e421ad75 lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x658:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f4c54 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 0 00010000:00010000:0.0:1775764770.637340:0:19862:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000030aeec1d/0x36f9bdc9e421ad75 lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x658:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5f4c54 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 0 00010000:00010000:0.0:1775764770.637347:0:19862:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000030aeec1d/0x36f9bdc9e421ad75 lrc: 4/0,0 mode: PW/PW res: [0x280000403:0x658:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f4c54 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 0 00000020:00010000:0.0:1775764770.637355:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_release():245 00000020:00010000:0.0:1775764770.637357:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.637359:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0df730 (ffff8f9966dc2128/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.637362:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764770.637364:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0df730 (ffff8f9966dc2368/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764770.637375:0:19862:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xec8:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764770.637392:0:19862:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2c1:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1775764770.637396:0:19862:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2c1:0x0] 00000008:00010000:0.0:1775764770.637420:0:19862:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:0.0:1775764770.637423:0:19862:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0df730, osc lock ffff8f996686f430, flags 0x400041012 00010000:00010000:0.0:1775764770.637431:0:19862:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f82f5f70/0x36f9bdc9e41cac05 lrc: 3/0,1 mode: PW/PW res: [0x240000403:0x57:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eea4a expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 49 lru_type: 1 00010000:00010000:0.0:1775764770.637440:0:19862:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f82f5f70/0x36f9bdc9e41cac05 lrc: 3/0,1 mode: PW/PW res: [0x240000403:0x57:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eea4a expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 49 lru_type: 1 00010000:00010000:0.0:1775764770.637448:0:19862:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f82f5f70/0x36f9bdc9e41cac05 lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x57:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4eea4a expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 49 lru_type: 1 00010000:00010000:0.0:1775764770.637455:0:19862:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f82f5f70/0x36f9bdc9e41cac05 lrc: 4/0,0 mode: PW/PW res: [0x240000403:0x57:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eea4a expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 50 lru_type: 1 00000020:00010000:0.0:1775764770.637464:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_release():245 00000020:00010000:0.0:1775764770.637466:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.637469:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0df730 (ffff8f9966dc2128/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.637472:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764770.637474:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0df730 (ffff8f9966dc2368/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764770.637484:0:19862:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2c1:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764770.637502:0:19862:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2db:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1775764770.637506:0:19862:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2db:0x0] 00000008:00010000:0.0:1775764770.637517:0:19862:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:0.0:1775764770.637519:0:19862:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0df730, osc lock ffff8f996686f430, flags 0x400041012 00010000:00010000:0.0:1775764770.637528:0:19862:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000054b8cc9/0x36f9bdc9e41caf93 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x64:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef2af expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 22 lru_type: 0 00010000:00010000:0.0:1775764770.637537:0:19862:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000054b8cc9/0x36f9bdc9e41caf93 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x64:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef2af expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 22 lru_type: 0 00010000:00010000:0.0:1775764770.637544:0:19862:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000054b8cc9/0x36f9bdc9e41caf93 lrc: 4/0,1 mode: PW/PW res: [0x240000403:0x64:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ef2af expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 22 lru_type: 0 00010000:00010000:0.0:1775764770.637551:0:19862:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000054b8cc9/0x36f9bdc9e41caf93 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x64:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef2af expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 0 00000020:00010000:0.0:1775764770.637561:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_release():245 00000020:00010000:0.0:1775764770.637563:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.637566:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0df730 (ffff8f9966dc2128/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.637568:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764770.637570:0:19862:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0df730 (ffff8f9966dc2368/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764770.637581:0:19862:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2db:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:0.0:1775764770.646369:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764770.646832:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978358016 00000100:00100000:0.0:1775764770.646841:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978358016 00000100:00100000:0.0:1775764770.646890:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24102 00000100:00100000:0.0:1775764770.646899:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e7100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+5:9373:x1862022978358016:12345-192.168.202.146@tcp:105: 00010000:00010000:0.0:1775764770.646912:0:5561:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c371 lrc: 5/1,0 mode: --/PR res: [0x200000401:0x291:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8fe5 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.646928:0:5561:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c371 lrc: 5/1,0 mode: --/PR res: [0x200000401:0x291:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8fe5 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.646936:0:5561:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c371 lrc: 5/1,0 mode: --/PR res: [0x200000401:0x291:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8fe5 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.647185:0:5561:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c371 lrc: 5/1,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x10080000000000 nid: local remote: 0x2aac6a50e5f8fe5 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.647195:0:5561:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c371 lrc: 6/1,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x10080000000000 nid: local remote: 0x2aac6a50e5f8fe5 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.647204:0:5561:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c371 lrc: 6/1,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x80000000000 nid: local remote: 0x2aac6a50e5f8fe5 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.647229:0:5561:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8f99435d4800) 00000100:00100000:0.0:1775764770.647286:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e7100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+5:9373:x1862022978358016:12345-192.168.202.146@tcp:105: Request processed in 387us (920us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764770.647294:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24102 00000100:00100000:0.0:1775764770.656427:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966915840:192.168.202.146@tcp:36:dbench.0 00000080:00200000:0.0:1775764770.656521:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x293:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764770.656535:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1775764770.656557:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:0.0:1775764770.656559:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:0.0:1775764770.656591:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x293:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764770.657051:0:19723:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x293:0x0](ffff8f9952dc9988) 00000008:00010000:0.0:1775764770.657090:0:19723:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 515 [cma]time 1775764732 1775764732 0 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a988520c/0x36f9bdc9e41ca598 lrc: 1/0,0 mode: PW/PW res: [0x240000403:0x43:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ed964 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 92 lru_type: 1 00000080:00200000:0.0:1775764770.657172:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.657178:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.657184:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:0.0:1775764770.657189:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.657192:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:0.0:1775764770.657195:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.657198:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:0.0:1775764770.657201:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.657204:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=COREL, flags=17 00000080:00200000:0.0:1775764770.657206:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x291:0x0](ffff8f9952dccb08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.657218:0:19723:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=GRAPHIC1.CDR, dir=[0x200000401:0x291:0x0](ffff8f9952dccb08), flags=1 00000080:00200000:0.0:1775764770.657222:0:19723:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=GRAPHIC1.CDR, dir=[0x200000401:0x291:0x0](ffff8f9952dccb08), intent=getattr 00000002:00010000:0.0:1775764770.657248:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: GRAPHIC1.CDR,[0x0:0x0:0x0]) in obj [0x200000401:0x291:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764770.657295:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c378 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x291:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.657303:0:19723:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c378 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x291:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.657313:0:19723:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c378 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x291:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764770.657333:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966916480:192.168.202.146@tcp:101:dbench.0 00000100:00100000:0.0:1775764770.657404:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f540 going to sleep for 11 seconds 00000100:00100000:0.0:1775764770.665835:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966916480:192.168.202.146@tcp:101:dbench.0 00010000:00010000:0.0:1775764770.665850:0:19723:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c378 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x291:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f90e8 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.665861:0:19723:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c378 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x299:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f90e8 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.665868:0:19723:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c378 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x299:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f90e8 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.665878:0:19723:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c378 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x299:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f90e8 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.665887:0:19723:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c378 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x299:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f90e8 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.665896:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c378 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x299:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f90e8 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.665903:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c378 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x299:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f90e8 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.665909:0:19723:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c378 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x299:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f90e8 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764770.665919:0:19723:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994f2e6d80 x1862022966916480/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/928 e 0 to 0 dl 1775764786 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'dbench.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1775764770.665928:0:19723:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c378 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x299:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f90e8 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:0.0:1775764770.665943:0:19723:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c378 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x299:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f90e8 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.665953:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093d000/0x36f9bdc9e4217671 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x299:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e9088 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.665962:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093d000/0x36f9bdc9e4217671 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x299:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e9088 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.665969:0:19723:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c378 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x299:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f90e8 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.665975:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c378 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x299:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x2aac6a50e5f90e8 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.665981:0:19723:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c378 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x299:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x2aac6a50e5f90e8 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.666034:0:19723:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000080:00200000:0.0:1775764770.666040:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x291:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1775764770.666101:0:19723:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x299:0x0](ffff8f9952d642c8): rc = 0 00000080:00200000:0.0:1775764770.666120:0:19723:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x299:0x0]: layout version change: 3 -> 3 00000080:00010000:0.0:1775764770.666129:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x299:0x0](ffff8f9952d642c8) for lock 0x36f9bdc9e4217671 00000080:00010000:0.0:1775764770.666139:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x299:0x0](ffff8f9952d642c8) 00000080:00010000:0.0:1775764770.666142:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e4217671 from it ffffafbac1c7bb20 00010000:00010000:0.0:1775764770.666145:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093d000/0x36f9bdc9e4217671 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x299:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5e9088 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.666158:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093d000/0x36f9bdc9e4217671 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x299:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e9088 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200008:0.0:1775764770.666196:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file GRAPHIC1.CDR:[0x200000401:0x299:0x0](ffff8f9952d642c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764770.666202:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x299:0x0](ffff8f9952d642c8),name=GRAPHIC1.CDR 00000002:00010000:0.0:1775764770.666212:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x299:0x0]) in obj [0x200000401:0x291:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764770.666219:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093d000/0x36f9bdc9e4217671 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x299:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e9088 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764770.666227:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093d000/0x36f9bdc9e4217671 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x299:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e9088 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764770.666232:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x291:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764770.666236:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x299:0x0](ffff8f9952d642c8) 00000080:00010000:0.0:1775764770.666238:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x299:0x0](ffff8f9952d642c8) for lock 0x36f9bdc9e4217671 00000080:00010000:0.0:1775764770.666241:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e4217671 from it ffffafbac1c7bcb8 00010000:00010000:0.0:1775764770.666244:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093d000/0x36f9bdc9e4217671 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x299:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5e9088 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764770.666250:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093d000/0x36f9bdc9e4217671 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x299:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e9088 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1775764770.666271:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x299:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1775764770.666277:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x299:0x0] 00000008:00010000:0.0:1775764770.666297:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1775764770.666299:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0df730, osc lock ffff8f996686f430, flags 0x400001002 00010000:00010000:0.0:1775764770.666313:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000006afc0c20/0x36f9bdc9e41ca66a lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x46:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4edb86 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 41 lru_type: 1 00010000:00010000:0.0:1775764770.666322:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000006afc0c20/0x36f9bdc9e41ca66a lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x46:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4edb86 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 41 lru_type: 1 00000020:00200000:0.0:1775764770.666331:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1775764770.666334:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000006afc0c20/0x36f9bdc9e41ca66a lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x46:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4edb86 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 41 lru_type: 1 00010000:00010000:0.0:1775764770.666341:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000006afc0c20/0x36f9bdc9e41ca66a lrc: 4/0,0 mode: PW/PW res: [0x240000403:0x46:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4edb86 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 42 lru_type: 1 00000020:00200000:0.0:1775764770.666350:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1775764770.666357:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x299:0x0] updating i_size 260096 i_blocks 1 00000020:00010000:0.0:1775764770.666360:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_release():245 00000020:00010000:0.0:1775764770.666363:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.666366:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0df730 (ffff8f9966dc2128/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.666369:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764770.666371:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0df730 (ffff8f9966dc2368/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764770.666385:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x299:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764770.666412:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.666416:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.666420:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:0.0:1775764770.666424:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.666427:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=16 00000080:00200000:0.0:1775764770.666429:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.666433:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:0.0:1775764770.666435:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.666438:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=COREL, flags=16 00000080:00200000:0.0:1775764770.666441:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x291:0x0](ffff8f9952dccb08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.666445:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=GRAPHIC1.CDR, flags=0 00000080:00200000:0.0:1775764770.666448:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x291:0x0](ffff8f9952dccb08), inode mode 41c0 mask 3 00000080:00200000:0.0:1775764770.666452:0:19723:0:(namei.c:2395:ll_unlink()) VFS Op:name=GRAPHIC1.CDR, dir=[0x200000401:0x291:0x0](ffff8f9952dccb08) 00010000:00010000:0.0:1775764770.666482:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c378 lrc: 3/0,0 mode: PR/PR res: [0x200000401:0x299:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f90e8 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1775764770.666501:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x299:0x0]: losing layout lock 00010000:00010000:0.0:1775764770.666516:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093d000/0x36f9bdc9e4217671 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x299:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5e9088 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1775764770.666532:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x299:0x0]: losing layout lock 00010000:00010000:0.0:1775764770.666555:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c378 lrc: 2/0,0 mode: --/PR res: [0x200000401:0x299:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f90e8 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.666563:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093d000/0x36f9bdc9e4217671 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x299:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5e9088 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764770.666569:0:19723:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1775764770.666571:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093d000/0x36f9bdc9e4217671 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x299:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5e9088 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000020:00100000:0.0:1775764770.666586:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 2 is allocated opc 36, max 7 00000100:00100000:0.0:1775764770.666592:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e6d80 x1862022966917248/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764770.666608:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966917248:192.168.202.146@tcp:36:dbench.0 00000100:00100000:0.0:1775764770.666683:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16fb40 going to sleep for 11 seconds 00000100:00100000:1.0:1775764770.669885:0:19863:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703df480 pname:cluuid:pid:xid:nid:opc:job ll_sa_19723:3f83546c-01a9-4156-8a97-44c446245d3e:19863:1862022966916352:192.168.202.146@tcp:37:ll_sa.0 00010000:00010000:1.0:1775764770.670479:0:19863:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c371 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8fe5 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.670495:0:19863:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c371 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8fe5 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000100:00100000:1.0:1775764770.670561:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764770.670761:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978358144 00000100:00100000:1.0:1775764770.670768:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978358144 00000100:00100000:1.0:1775764770.670819:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24103 00000100:00100000:1.0:1775764770.670829:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970c02d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+5:5795:x1862022978358144:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764770.670843:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c371 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x2aac6a50e5f8fe5 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.670898:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764770.670905:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970c02d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+5:5795:x1862022978358144:12345-192.168.202.146@tcp:104: Request processed in 78us (344us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764770.670912:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24103 00010000:00010000:1.0:1775764770.671013:0:19863:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081c00/0x36f9bdc9e421c37f lrc: 3/1,0 mode: --/CR res: [0x200000401:0x291:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 19863 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.671020:0:19863:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081c00/0x36f9bdc9e421c37f lrc: 3/1,0 mode: --/CR res: [0x200000401:0x291:0x0].0x0 bits 0x3/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19863 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.673146:0:19863:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081200/0x36f9bdc9e421c386 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x291:0x0].0x0 bits 0x0/0x0 rrc: 6 type: IBT flags: 0x10000000000000 pid: 19863 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.673155:0:19863:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081200/0x36f9bdc9e421c386 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x291:0x0].0x0 bits 0x3/0x0 rrc: 6 type: IBT flags: 0x0 pid: 19863 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.673196:0:19863:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081800/0x36f9bdc9e421c38d lrc: 3/1,0 mode: --/CR res: [0x200000401:0x291:0x0].0x0 bits 0x0/0x0 rrc: 7 type: IBT flags: 0x10000000000000 pid: 19863 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.673204:0:19863:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081800/0x36f9bdc9e421c38d lrc: 3/1,0 mode: --/CR res: [0x200000401:0x291:0x0].0x0 bits 0x3/0x0 rrc: 8 type: IBT flags: 0x0 pid: 19863 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.673219:0:19863:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081400/0x36f9bdc9e421c394 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x291:0x0].0x0 bits 0x0/0x0 rrc: 7 type: IBT flags: 0x10000000000000 pid: 19863 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.673225:0:19863:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081400/0x36f9bdc9e421c394 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x291:0x0].0x0 bits 0x3/0x0 rrc: 7 type: IBT flags: 0x0 pid: 19863 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.673238:0:19863:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080400/0x36f9bdc9e421c39b lrc: 3/1,0 mode: --/CR res: [0x200000401:0x291:0x0].0x0 bits 0x0/0x0 rrc: 8 type: IBT flags: 0x10000000000000 pid: 19863 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.673245:0:19863:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080400/0x36f9bdc9e421c39b lrc: 3/1,0 mode: --/CR res: [0x200000401:0x291:0x0].0x0 bits 0x3/0x0 rrc: 8 type: IBT flags: 0x0 pid: 19863 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.673258:0:19863:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080800/0x36f9bdc9e421c3a2 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x291:0x0].0x0 bits 0x0/0x0 rrc: 9 type: IBT flags: 0x10000000000000 pid: 19863 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.673263:0:19863:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080800/0x36f9bdc9e421c3a2 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x291:0x0].0x0 bits 0x3/0x0 rrc: 9 type: IBT flags: 0x0 pid: 19863 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.673277:0:19863:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081600/0x36f9bdc9e421c3a9 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x291:0x0].0x0 bits 0x0/0x0 rrc: 10 type: IBT flags: 0x10000000000000 pid: 19863 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.673284:0:19863:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081600/0x36f9bdc9e421c3a9 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x291:0x0].0x0 bits 0x3/0x0 rrc: 10 type: IBT flags: 0x0 pid: 19863 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.673301:0:19863:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080a00/0x36f9bdc9e421c3b0 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x291:0x0].0x0 bits 0x0/0x0 rrc: 11 type: IBT flags: 0x10000000000000 pid: 19863 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.673305:0:19863:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080a00/0x36f9bdc9e421c3b0 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x291:0x0].0x0 bits 0x3/0x0 rrc: 11 type: IBT flags: 0x0 pid: 19863 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764770.673338:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970c03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966917760:192.168.202.146@tcp:63:ll_sa.0 00010000:00010000:1.0:1775764770.680815:0:2379:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081c00/0x36f9bdc9e421c37f lrc: 4/1,0 mode: --/CR res: [0x200000401:0x291:0x0].0x0 bits 0x3/0x0 rrc: 11 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f913c expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.680838:0:2379:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081c00/0x36f9bdc9e421c37f lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2a0:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f913c expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.680847:0:2379:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081c00/0x36f9bdc9e421c37f lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2a0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f913c expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.680858:0:2379:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081c00/0x36f9bdc9e421c37f lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2a0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f913c expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.680868:0:2379:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081c00/0x36f9bdc9e421c37f lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2a0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f913c expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.680879:0:2379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081c00/0x36f9bdc9e421c37f lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2a0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f913c expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.680887:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081c00/0x36f9bdc9e421c37f lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2a0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f913c expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.680894:0:2379:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081c00/0x36f9bdc9e421c37f lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2a0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f913c expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764770.680908:0:2379:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970c03b80 x1862022966917760/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 3472/6120 e 0 to 0 dl 1775764786 ref 1 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:1.0:1775764770.680920:0:2379:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081c00/0x36f9bdc9e421c37f lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f913c expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:1.0:1775764770.680932:0:2379:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c37f from it ffff8f9970e08588 00010000:00010000:1.0:1775764770.680936:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081c00/0x36f9bdc9e421c37f lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f913c expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681016:0:2379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081c00/0x36f9bdc9e421c37f lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f913c expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.681044:0:2379:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081200/0x36f9bdc9e421c386 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x291:0x0].0x0 bits 0x3/0x0 rrc: 10 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f916d expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681053:0:2379:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081200/0x36f9bdc9e421c386 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x297:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f916d expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681059:0:2379:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081200/0x36f9bdc9e421c386 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x297:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f916d expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681066:0:2379:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081200/0x36f9bdc9e421c386 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x297:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f916d expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681073:0:2379:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081200/0x36f9bdc9e421c386 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x297:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f916d expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681081:0:2379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081200/0x36f9bdc9e421c386 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x297:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f916d expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681087:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081200/0x36f9bdc9e421c386 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x297:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f916d expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681093:0:2379:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081200/0x36f9bdc9e421c386 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x297:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f916d expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764770.681107:0:2379:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970c03b80 x1862022966917760/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 3472/6120 e 0 to 0 dl 1775764786 ref 2 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:1.0:1775764770.681115:0:2379:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081200/0x36f9bdc9e421c386 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x297:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f916d expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:1.0:1775764770.681122:0:2379:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c386 from it ffff8f9970e0bd88 00010000:00010000:1.0:1775764770.681125:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081200/0x36f9bdc9e421c386 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x297:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f916d expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681132:0:2379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081200/0x36f9bdc9e421c386 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x297:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f916d expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.681145:0:2379:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081800/0x36f9bdc9e421c38d lrc: 4/1,0 mode: --/CR res: [0x200000401:0x291:0x0].0x0 bits 0x3/0x0 rrc: 9 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9189 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681154:0:2379:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081800/0x36f9bdc9e421c38d lrc: 4/1,0 mode: --/PR res: [0x200000401:0x29b:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9189 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681161:0:2379:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081800/0x36f9bdc9e421c38d lrc: 4/1,0 mode: --/PR res: [0x200000401:0x29b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9189 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681168:0:2379:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081800/0x36f9bdc9e421c38d lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x29b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9189 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681175:0:2379:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081800/0x36f9bdc9e421c38d lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x29b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9189 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681186:0:2379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081800/0x36f9bdc9e421c38d lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x29b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9189 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681193:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081800/0x36f9bdc9e421c38d lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x29b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9189 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681199:0:2379:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081800/0x36f9bdc9e421c38d lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x29b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9189 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764770.681206:0:2379:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970c03b80 x1862022966917760/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 3472/6120 e 0 to 0 dl 1775764786 ref 3 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:1.0:1775764770.681215:0:2379:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081800/0x36f9bdc9e421c38d lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x29b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9189 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:1.0:1775764770.681224:0:2379:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c38d from it ffff8f9970e0b188 00010000:00010000:1.0:1775764770.681227:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081800/0x36f9bdc9e421c38d lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x29b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9189 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681233:0:2379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081800/0x36f9bdc9e421c38d lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x29b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9189 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.681244:0:2379:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081400/0x36f9bdc9e421c394 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x291:0x0].0x0 bits 0x3/0x0 rrc: 8 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f91a5 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681253:0:2379:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081400/0x36f9bdc9e421c394 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x296:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f91a5 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681264:0:2379:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081400/0x36f9bdc9e421c394 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x296:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f91a5 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681271:0:2379:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081400/0x36f9bdc9e421c394 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x296:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f91a5 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681279:0:2379:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081400/0x36f9bdc9e421c394 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x296:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f91a5 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681286:0:2379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081400/0x36f9bdc9e421c394 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x296:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f91a5 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681292:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081400/0x36f9bdc9e421c394 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x296:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f91a5 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681298:0:2379:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081400/0x36f9bdc9e421c394 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x296:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f91a5 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764770.681306:0:2379:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970c03b80 x1862022966917760/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 3472/6120 e 0 to 0 dl 1775764786 ref 4 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:1.0:1775764770.681315:0:2379:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081400/0x36f9bdc9e421c394 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x296:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f91a5 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:1.0:1775764770.681323:0:2379:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c394 from it ffff8f9970e0b588 00010000:00010000:1.0:1775764770.681326:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081400/0x36f9bdc9e421c394 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x296:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f91a5 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681337:0:2379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081400/0x36f9bdc9e421c394 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x296:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f91a5 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.681349:0:2379:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080400/0x36f9bdc9e421c39b lrc: 4/1,0 mode: --/CR res: [0x200000401:0x291:0x0].0x0 bits 0x3/0x0 rrc: 7 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f91c8 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681358:0:2379:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080400/0x36f9bdc9e421c39b lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2a6:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f91c8 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681364:0:2379:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080400/0x36f9bdc9e421c39b lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2a6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f91c8 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681371:0:2379:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080400/0x36f9bdc9e421c39b lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2a6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f91c8 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681378:0:2379:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080400/0x36f9bdc9e421c39b lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2a6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f91c8 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681385:0:2379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080400/0x36f9bdc9e421c39b lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2a6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f91c8 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681486:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080400/0x36f9bdc9e421c39b lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2a6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f91c8 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681492:0:2379:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080400/0x36f9bdc9e421c39b lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2a6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f91c8 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764770.681505:0:2379:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970c03b80 x1862022966917760/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 3472/6120 e 0 to 0 dl 1775764786 ref 5 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:1.0:1775764770.681512:0:2379:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080400/0x36f9bdc9e421c39b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f91c8 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:1.0:1775764770.681521:0:2379:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c39b from it ffff8f9970e08d88 00010000:00010000:1.0:1775764770.681524:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080400/0x36f9bdc9e421c39b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f91c8 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681530:0:2379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080400/0x36f9bdc9e421c39b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f91c8 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.681543:0:2379:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080800/0x36f9bdc9e421c3a2 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x291:0x0].0x0 bits 0x3/0x0 rrc: 6 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f91e4 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681550:0:2379:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080800/0x36f9bdc9e421c3a2 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2a2:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f91e4 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681557:0:2379:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080800/0x36f9bdc9e421c3a2 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2a2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f91e4 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681564:0:2379:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080800/0x36f9bdc9e421c3a2 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2a2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f91e4 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681571:0:2379:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080800/0x36f9bdc9e421c3a2 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2a2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f91e4 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681583:0:2379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080800/0x36f9bdc9e421c3a2 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2a2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f91e4 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681590:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080800/0x36f9bdc9e421c3a2 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2a2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f91e4 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681597:0:2379:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080800/0x36f9bdc9e421c3a2 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2a2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f91e4 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764770.681605:0:2379:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970c03b80 x1862022966917760/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 3472/6120 e 0 to 0 dl 1775764786 ref 6 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:1.0:1775764770.681613:0:2379:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080800/0x36f9bdc9e421c3a2 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f91e4 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:1.0:1775764770.681633:0:2379:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c3a2 from it ffff8f9970e09d88 00010000:00010000:1.0:1775764770.681636:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080800/0x36f9bdc9e421c3a2 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f91e4 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681643:0:2379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080800/0x36f9bdc9e421c3a2 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f91e4 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.681656:0:2379:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081600/0x36f9bdc9e421c3a9 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x291:0x0].0x0 bits 0x3/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9200 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681664:0:2379:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081600/0x36f9bdc9e421c3a9 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2a4:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9200 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681675:0:2379:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081600/0x36f9bdc9e421c3a9 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2a4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9200 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681682:0:2379:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081600/0x36f9bdc9e421c3a9 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2a4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9200 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681690:0:2379:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081600/0x36f9bdc9e421c3a9 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2a4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9200 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681697:0:2379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081600/0x36f9bdc9e421c3a9 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2a4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9200 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681703:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081600/0x36f9bdc9e421c3a9 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2a4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9200 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681710:0:2379:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081600/0x36f9bdc9e421c3a9 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2a4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9200 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764770.681719:0:2379:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970c03b80 x1862022966917760/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 3472/6120 e 0 to 0 dl 1775764786 ref 7 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:1.0:1775764770.681727:0:2379:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081600/0x36f9bdc9e421c3a9 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9200 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:1.0:1775764770.681736:0:2379:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c3a9 from it ffff8f9970e09188 00010000:00010000:1.0:1775764770.681784:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081600/0x36f9bdc9e421c3a9 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9200 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681797:0:2379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081600/0x36f9bdc9e421c3a9 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9200 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.681810:0:2379:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080a00/0x36f9bdc9e421c3b0 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x291:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9223 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681820:0:2379:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080a00/0x36f9bdc9e421c3b0 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2a8:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9223 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681827:0:2379:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080a00/0x36f9bdc9e421c3b0 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2a8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9223 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681834:0:2379:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080a00/0x36f9bdc9e421c3b0 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2a8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9223 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681842:0:2379:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080a00/0x36f9bdc9e421c3b0 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2a8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9223 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681850:0:2379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080a00/0x36f9bdc9e421c3b0 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2a8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9223 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681858:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080a00/0x36f9bdc9e421c3b0 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2a8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9223 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681865:0:2379:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080a00/0x36f9bdc9e421c3b0 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2a8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9223 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764770.681879:0:2379:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970c03b80 x1862022966917760/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 3472/6120 e 0 to 0 dl 1775764786 ref 8 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:1.0:1775764770.681887:0:2379:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080a00/0x36f9bdc9e421c3b0 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9223 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:1.0:1775764770.681895:0:2379:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c3b0 from it ffff8f9970e09988 00010000:00010000:1.0:1775764770.681898:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080a00/0x36f9bdc9e421c3b0 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9223 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.681903:0:2379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080a00/0x36f9bdc9e421c3b0 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9223 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000100:00100000:1.0:1775764770.681917:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970c03b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966917760:192.168.202.146@tcp:63:ll_sa.0 00010000:00010000:1.0:1775764770.682078:0:21:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081c00/0x36f9bdc9e421c37f lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f913c expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764770.682175:0:21:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x2a0:0x0](ffff8f9952da3248): rc = 0 00000080:00200000:1.0:1775764770.682196:0:21:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x2a0:0x0]: layout version change: 3 -> 3 00000080:00010000:1.0:1775764770.682205:0:21:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2a0:0x0](ffff8f9952da3248) for lock 0x36f9bdc9e421c37f 00000100:00100000:0.0:1775764770.683913:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970d9c380 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966917632:192.168.202.146@tcp:36:dbench.0 00000080:00200000:0.0:1775764770.683997:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2be:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764770.684009:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1775764770.684025:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:0.0:1775764770.684027:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:0.0:1775764770.684055:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2be:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764770.684520:0:19724:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2be:0x0](ffff8f9952e49148) 00000008:00010000:0.0:1775764770.684564:0:19724:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 515 [cma]time 1775764768 1775764768 0 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000076c55670/0x36f9bdc9e41cab87 lrc: 1/0,0 mode: PW/PW res: [0x280000403:0x55:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee8ec expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 34 lru_type: 1 00000080:00200000:0.0:1775764770.684674:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.684994:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.685002:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:0.0:1775764770.685032:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.685036:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:0.0:1775764770.685039:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.685042:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:0.0:1775764770.685045:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.685048:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:0.0:1775764770.685051:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.685067:0:19724:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=__52E2B.DB, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), flags=1 00000080:00200000:0.0:1775764770.685071:0:19724:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=__52E2B.DB, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), intent=getattr 00010000:00010000:0.0:1775764770.685088:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98200/0x36f9bdc9e421c2f3 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xed7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8e64 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:0.0:1775764770.685102:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c2f3 from it ffffafbac1c5b900 00010000:00010000:0.0:1775764770.685106:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98200/0x36f9bdc9e421c2f3 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xed7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8e64 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764770.685113:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98200/0x36f9bdc9e421c2f3 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xed7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8e64 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:0.0:1775764770.685135:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file __52E2B.DB:[0x200000401:0xed7:0x0](ffff8f9952eb4b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764770.685140:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xed7:0x0](ffff8f9952eb4b08),name=__52E2B.DB 00000002:00010000:0.0:1775764770.685156:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xed7:0x0]) in obj [0x200000401:0x2b8:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764770.685164:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5800/0x36f9bdc9e421c14f lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xed7:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f886e expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.685172:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5800/0x36f9bdc9e421c14f lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xed7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f886e expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764770.685179:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2b8:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764770.685184:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xed7:0x0](ffff8f9952eb4b08) 00000080:00010000:0.0:1775764770.685187:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xed7:0x0](ffff8f9952eb4b08) for lock 0x36f9bdc9e421c14f 00000080:00010000:0.0:1775764770.685192:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c14f from it ffffafbac1c5bcb8 00010000:00010000:0.0:1775764770.685195:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5800/0x36f9bdc9e421c14f lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xed7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f886e expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.685202:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5800/0x36f9bdc9e421c14f lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xed7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f886e expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764770.685224:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xed7:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1775764770.685228:0:19724:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xed7:0x0] 00000008:00010000:0.0:1775764770.685240:0:19724:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1775764770.685241:0:19724:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0df730, osc lock ffff8f996686f430, flags 0x400001002 00010000:00010000:0.0:1775764770.685251:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4034b9e/0x36f9bdc9e421c125 lrc: 3/0,1 mode: PW/PW res: [0x240000403:0x660:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f87f7 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764770.685257:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4034b9e/0x36f9bdc9e421c125 lrc: 3/0,1 mode: PW/PW res: [0x240000403:0x660:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f87f7 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:0.0:1775764770.685265:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1775764770.685267:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4034b9e/0x36f9bdc9e421c125 lrc: 6/0,1 mode: PW/PW res: [0x240000403:0x660:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5f87f7 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764770.685272:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4034b9e/0x36f9bdc9e421c125 lrc: 5/0,0 mode: PW/PW res: [0x240000403:0x660:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f87f7 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:0.0:1775764770.685278:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1775764770.685283:0:19724:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0xed7:0x0] updating i_size 2048 i_blocks 1 00000020:00010000:0.0:1775764770.685285:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_release():245 00000020:00010000:0.0:1775764770.685287:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.685288:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0df730 (ffff8f9966dc2128/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.685290:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764770.685292:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0df730 (ffff8f9966dc2368/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764770.685301:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xed7:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764770.685314:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.685316:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.685318:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:0.0:1775764770.685320:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.685322:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=16 00000080:00200000:0.0:1775764770.685324:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.685326:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:0.0:1775764770.685328:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.685329:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=16 00000080:00200000:0.0:1775764770.685331:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.685334:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=__52E2B.DB, flags=0 00000080:00200000:0.0:1775764770.685336:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 3 00000080:00200000:0.0:1775764770.685339:0:19724:0:(namei.c:2395:ll_unlink()) VFS Op:name=__52E2B.DB, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8) 00010000:00010000:0.0:1775764770.685365:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98200/0x36f9bdc9e421c2f3 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xed7:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f8e64 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1775764770.685380:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0xed7:0x0]: losing layout lock 00010000:00010000:0.0:1775764770.685407:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5800/0x36f9bdc9e421c14f lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xed7:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f886e expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764770.685417:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0xed7:0x0]: losing layout lock 00010000:00010000:0.0:1775764770.685447:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98200/0x36f9bdc9e421c2f3 lrc: 1/0,0 mode: --/PR res: [0x200000401:0xed7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f8e64 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764770.685451:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5800/0x36f9bdc9e421c14f lrc: 1/0,0 mode: --/PR res: [0x200000401:0xed7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f886e expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764770.685455:0:19724:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1775764770.685457:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98200/0x36f9bdc9e421c2f3 lrc: 0/0,0 mode: --/PR res: [0x200000401:0xed7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f8e64 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764770.685463:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5800/0x36f9bdc9e421c14f lrc: 0/0,0 mode: --/PR res: [0x200000401:0xed7:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f886e expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000020:00100000:0.0:1775764770.685477:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1775764770.685484:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e5180 x1862022966918016/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764770.685502:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966918016:192.168.202.146@tcp:36:dbench.0 00000100:00100000:0.0:1775764770.685686:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f000 going to sleep for 11 seconds 00000080:00010000:1.0:1775764770.689881:0:21:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c37f from it ffff8f9970e08588 00010000:00010000:1.0:1775764770.689892:0:21:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081c00/0x36f9bdc9e421c37f lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f913c expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.689903:0:21:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081c00/0x36f9bdc9e421c37f lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f913c expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764770.690039:0:21:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081200/0x36f9bdc9e421c386 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x297:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f916d expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764770.690110:0:21:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x297:0x0](ffff8f9952dce3c8): rc = 0 00000080:00200000:1.0:1775764770.690131:0:21:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x297:0x0]: layout version change: 3 -> 3 00000080:00010000:1.0:1775764770.690142:0:21:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x297:0x0](ffff8f9952dce3c8) for lock 0x36f9bdc9e421c386 00000080:00010000:1.0:1775764770.690161:0:21:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c386 from it ffff8f9970e0bd88 00010000:00010000:1.0:1775764770.690165:0:21:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081200/0x36f9bdc9e421c386 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x297:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f916d expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.690172:0:21:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081200/0x36f9bdc9e421c386 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x297:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f916d expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764770.690189:0:21:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081800/0x36f9bdc9e421c38d lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x29b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9189 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764770.690221:0:21:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x29b:0x0](ffff8f9952dcec08): rc = 0 00000080:00200000:1.0:1775764770.690236:0:21:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x29b:0x0]: layout version change: 3 -> 3 00000080:00010000:1.0:1775764770.690244:0:21:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x29b:0x0](ffff8f9952dcec08) for lock 0x36f9bdc9e421c38d 00000080:00010000:1.0:1775764770.690248:0:21:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c38d from it ffff8f9970e0b188 00010000:00010000:1.0:1775764770.690251:0:21:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081800/0x36f9bdc9e421c38d lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x29b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9189 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.690258:0:21:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081800/0x36f9bdc9e421c38d lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x29b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9189 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764770.690278:0:21:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081400/0x36f9bdc9e421c394 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x296:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f91a5 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764770.690307:0:21:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x296:0x0](ffff8f9952dcdb88): rc = 0 00000080:00200000:1.0:1775764770.690321:0:21:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x296:0x0]: layout version change: 3 -> 3 00000080:00010000:1.0:1775764770.690329:0:21:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x296:0x0](ffff8f9952dcdb88) for lock 0x36f9bdc9e421c394 00000080:00010000:1.0:1775764770.690332:0:21:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c394 from it ffff8f9970e0b588 00010000:00010000:1.0:1775764770.690335:0:21:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081400/0x36f9bdc9e421c394 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x296:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f91a5 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.691830:0:19863:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c3b7 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x291:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 19863 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.691839:0:19863:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c3b7 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x291:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19863 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.692873:0:21:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081400/0x36f9bdc9e421c394 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x296:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f91a5 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764770.692901:0:21:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080400/0x36f9bdc9e421c39b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f91c8 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764770.693082:0:21:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x2a6:0x0](ffff8f9952d10908): rc = 0 00000100:00100000:0.0:1775764770.693819:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966918400:192.168.202.146@tcp:63:ll_sa.0 00000080:00200000:1.0:1775764770.694262:0:21:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x2a6:0x0]: layout version change: 3 -> 3 00000080:00010000:1.0:1775764770.694278:0:21:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2a6:0x0](ffff8f9952d10908) for lock 0x36f9bdc9e421c39b 00000080:00010000:1.0:1775764770.694289:0:21:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c39b from it ffff8f9970e08d88 00010000:00010000:1.0:1775764770.694295:0:21:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080400/0x36f9bdc9e421c39b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f91c8 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.694305:0:21:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080400/0x36f9bdc9e421c39b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f91c8 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764770.694340:0:21:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080800/0x36f9bdc9e421c3a2 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f91e4 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764770.694374:0:21:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x2a2:0x0](ffff8f9952dcf448): rc = 0 00000080:00200000:1.0:1775764770.694390:0:21:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x2a2:0x0]: layout version change: 3 -> 3 00000080:00010000:1.0:1775764770.694398:0:21:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2a2:0x0](ffff8f9952dcf448) for lock 0x36f9bdc9e421c3a2 00000080:00010000:1.0:1775764770.694403:0:21:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c3a2 from it ffff8f9970e09d88 00010000:00010000:1.0:1775764770.694406:0:21:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080800/0x36f9bdc9e421c3a2 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f91e4 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.694414:0:21:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080800/0x36f9bdc9e421c3a2 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f91e4 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764770.694430:0:21:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081600/0x36f9bdc9e421c3a9 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9200 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764770.694463:0:21:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x2a4:0x0](ffff8f9952d88908): rc = 0 00000080:00200000:1.0:1775764770.694477:0:21:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x2a4:0x0]: layout version change: 3 -> 3 00000080:00010000:1.0:1775764770.694485:0:21:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2a4:0x0](ffff8f9952d88908) for lock 0x36f9bdc9e421c3a9 00000080:00010000:1.0:1775764770.694489:0:21:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c3a9 from it ffff8f9970e09188 00010000:00010000:1.0:1775764770.694492:0:21:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081600/0x36f9bdc9e421c3a9 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9200 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.694499:0:21:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081600/0x36f9bdc9e421c3a9 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9200 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764770.696856:0:21:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080a00/0x36f9bdc9e421c3b0 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9223 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.697033:0:2380:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c3b7 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x291:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9262 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.697047:0:2380:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c3b7 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x29e:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9262 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.697055:0:2380:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c3b7 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x29e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9262 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.697065:0:2380:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c3b7 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x29e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9262 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.697074:0:2380:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c3b7 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x29e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9262 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.697083:0:2380:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c3b7 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x29e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9262 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.697096:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c3b7 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x29e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9262 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.697103:0:2380:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c3b7 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x29e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9262 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764770.697118:0:2380:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994f2e4380 x1862022966918400/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 672/968 e 0 to 0 dl 1775764786 ref 1 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1775764770.697127:0:2380:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c3b7 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x29e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9262 expref: -99 pid: 19863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1775764770.697137:0:2380:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c3b7 from it ffff8f9966bf7988 00010000:00010000:0.0:1775764770.697140:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c3b7 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x29e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9262 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.697146:0:2380:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c3b7 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x29e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9262 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1775764770.697171:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966918400:192.168.202.146@tcp:63:ll_sa.0 00010000:00010000:0.0:1775764770.697196:0:131:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c3b7 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x29e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9262 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764770.697243:0:131:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x29e:0x0](ffff8f9952dcaa08): rc = 0 00000080:00200000:0.0:1775764770.697260:0:131:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x29e:0x0]: layout version change: 3 -> 3 00000080:00010000:0.0:1775764770.697269:0:131:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x29e:0x0](ffff8f9952dcaa08) for lock 0x36f9bdc9e421c3b7 00000080:00010000:0.0:1775764770.697278:0:131:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c3b7 from it ffff8f9966bf7988 00010000:00010000:0.0:1775764770.697281:0:131:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c3b7 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x29e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9262 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.697288:0:131:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c3b7 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x29e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9262 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764770.698583:0:21:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x2a8:0x0](ffff8f9952d8cb08): rc = 0 00000080:00200000:1.0:1775764770.698633:0:21:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x2a8:0x0]: layout version change: 3 -> 3 00000080:00010000:1.0:1775764770.698644:0:21:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2a8:0x0](ffff8f9952d8cb08) for lock 0x36f9bdc9e421c3b0 00000080:00010000:1.0:1775764770.698652:0:21:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c3b0 from it ffff8f9970e09988 00010000:00010000:1.0:1775764770.698656:0:21:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080a00/0x36f9bdc9e421c3b0 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9223 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.698666:0:21:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080a00/0x36f9bdc9e421c3b0 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9223 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000100:00100000:1.0:1775764770.714327:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703df480 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966918656:192.168.202.146@tcp:36:dbench.0 00000080:00200000:0.0:1775764770.714588:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2df:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764770.714600:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1775764770.714718:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:0.0:1775764770.714720:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:0.0:1775764770.714849:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2df:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764770.715264:0:19724:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2df:0x0](ffff8f9952dd3a88) 00000008:00010000:0.0:1775764770.715305:0:19724:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 1 [cma]time 1775764059 1775764059 0 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001ba52963/0x36f9bdc9e41cb011 lrc: 1/0,0 mode: PW/PW res: [0x240000403:0x66:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef3dc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 1 00000080:00200000:0.0:1775764770.715486:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.715493:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.715499:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:0.0:1775764770.715504:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.715508:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:0.0:1775764770.715511:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.715515:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:0.0:1775764770.715517:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.715521:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:0.0:1775764770.715523:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.715537:0:19724:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=STUDENTS.XG0, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), flags=1 00000080:00200000:0.0:1775764770.715542:0:19724:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=STUDENTS.XG0, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), intent=getattr 00010000:00010000:0.0:1775764770.715560:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4600/0x36f9bdc9e421c308 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8eb1 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:0.0:1775764770.715574:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c308 from it ffffafbac1c5b900 00010000:00010000:0.0:1775764770.715578:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4600/0x36f9bdc9e421c308 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8eb1 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764770.715588:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4600/0x36f9bdc9e421c308 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8eb1 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:0.0:1775764770.715708:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file STUDENTS.XG0:[0x200000401:0x2c5:0x0](ffff8f9952d8ec08), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764770.715714:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2c5:0x0](ffff8f9952d8ec08),name=STUDENTS.XG0 00000002:00010000:0.0:1775764770.715729:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2c5:0x0]) in obj [0x200000401:0x2b8:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764770.715823:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421bf1f lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2c5:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f81f3 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.715836:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421bf1f lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2c5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f81f3 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764770.715845:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2b8:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764770.715851:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2c5:0x0](ffff8f9952d8ec08) 00000080:00010000:0.0:1775764770.715854:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2c5:0x0](ffff8f9952d8ec08) for lock 0x36f9bdc9e421bf1f 00000080:00010000:0.0:1775764770.715859:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421bf1f from it ffffafbac1c5bcb8 00010000:00010000:0.0:1775764770.715862:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421bf1f lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f81f3 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.715870:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421bf1f lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f81f3 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764770.715901:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2c5:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1775764770.715910:0:19724:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2c5:0x0] 00000008:00010000:0.0:1775764770.715933:0:19724:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1775764770.715936:0:19724:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0dfeb0, osc lock ffff8f996686f430, flags 0x400001002 00010000:00010000:3.0:1775764770.721059:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970973600/0x36f9bdc9e4217c8a lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x296:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5ea438 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764770.721081:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x296:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764770.721086:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x296:0x0] 00000008:00010000:3.0:1775764770.721100:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1775764770.721103:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d5cb0, osc lock ffff8f9966acef70, flags 0x400001002 00010000:00010000:3.0:1775764770.721116:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f22973fd/0x36f9bdc9e41ca5ec lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x44:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eda44 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 92 lru_type: 1 00010000:00010000:3.0:1775764770.721124:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f22973fd/0x36f9bdc9e41ca5ec lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x44:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eda44 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 92 lru_type: 1 00000020:00200000:3.0:1775764770.721131:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1775764770.721133:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f22973fd/0x36f9bdc9e41ca5ec lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x44:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4eda44 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 92 lru_type: 1 00010000:00010000:3.0:1775764770.721138:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f22973fd/0x36f9bdc9e41ca5ec lrc: 4/0,0 mode: PW/PW res: [0x280000403:0x44:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eda44 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 93 lru_type: 1 00000020:00200000:3.0:1775764770.721144:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1775764770.721151:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x296:0x0] updating i_size 260096 i_blocks 515 00000020:00010000:3.0:1775764770.721153:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef50db0 (ffff8f99703141b8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764770.721155:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef50db0 (ffff8f99703141b8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.721157:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d5cb0 (ffff8f9970314128/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.721160:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef50db0 (ffff8f99703141b8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764770.721161:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d5cb0 (ffff8f99703141b8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764770.721171:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x296:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764770.721184:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.721187:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.721189:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:3.0:1775764770.721191:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.721193:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=16 00000080:00200000:3.0:1775764770.721194:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.721196:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:3.0:1775764770.721197:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.721199:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=COREL, flags=16 00000080:00200000:3.0:1775764770.721200:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x291:0x0](ffff8f9952dccb08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.721203:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=CDRBARS.CFG, flags=0 00000080:00200000:3.0:1775764770.721205:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x291:0x0](ffff8f9952dccb08), inode mode 41c0 mask 3 00000080:00200000:3.0:1775764770.721208:0:19723:0:(namei.c:2395:ll_unlink()) VFS Op:name=CDRBARS.CFG, dir=[0x200000401:0x291:0x0](ffff8f9952dccb08) 00010000:00010000:3.0:1775764770.721232:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081400/0x36f9bdc9e421c394 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x296:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f91a5 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764770.721245:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x296:0x0]: losing layout lock 00010000:00010000:3.0:1775764770.721258:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970973600/0x36f9bdc9e4217c8a lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x296:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5ea438 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764770.721266:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x296:0x0]: losing layout lock 00010000:00010000:3.0:1775764770.721293:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081400/0x36f9bdc9e421c394 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x296:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f91a5 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764770.721298:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970973600/0x36f9bdc9e4217c8a lrc: 1/0,0 mode: --/PR res: [0x200000401:0x296:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5ea438 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764770.721301:0:19723:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1775764770.721303:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081400/0x36f9bdc9e421c394 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x296:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f91a5 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764770.721314:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970973600/0x36f9bdc9e4217c8a lrc: 0/0,0 mode: --/PR res: [0x200000401:0x296:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5ea438 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000020:00100000:3.0:1775764770.721329:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1775764770.721335:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970eddf80 x1862022966919424/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764770.721351:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966919424:192.168.202.146@tcp:36:dbench.0 00000100:00100000:3.0:1775764770.721405:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99707ef6c0 going to sleep for 11 seconds 00010000:00010000:0.0:1775764770.723275:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ac098a84/0x36f9bdc9e41cac83 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x59:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eeb54 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 0 00010000:00010000:0.0:1775764770.723319:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ac098a84/0x36f9bdc9e41cac83 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x59:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eeb54 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 0 00000020:00200000:0.0:1775764770.723334:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1775764770.723340:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ac098a84/0x36f9bdc9e41cac83 lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x59:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4eeb54 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 0 00010000:00010000:0.0:1775764770.723350:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ac098a84/0x36f9bdc9e41cac83 lrc: 4/0,0 mode: PW/PW res: [0x240000403:0x59:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eeb54 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 21 lru_type: 0 00000020:00200000:0.0:1775764770.723365:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1775764770.723378:0:19724:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2c5:0x0] updating i_size 260096 i_blocks 1 00000020:00010000:0.0:1775764770.723383:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_release():245 00000020:00010000:0.0:1775764770.723386:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.723391:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0dfeb0 (ffff8f9966dc2128/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.723394:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764770.723397:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0dfeb0 (ffff8f9966dc2368/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764770.723435:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2c5:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764770.723473:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.723479:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.723485:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:0.0:1775764770.723491:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.723495:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=16 00000080:00200000:0.0:1775764770.723498:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.723501:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:0.0:1775764770.723503:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.723507:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=16 00000080:00200000:0.0:1775764770.723509:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.723514:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=STUDENTS.XG0, flags=0 00000080:00200000:0.0:1775764770.723517:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 3 00000080:00200000:0.0:1775764770.723522:0:19724:0:(namei.c:2395:ll_unlink()) VFS Op:name=STUDENTS.XG0, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8) 00010000:00010000:0.0:1775764770.723578:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4600/0x36f9bdc9e421c308 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c5:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f8eb1 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1775764770.723598:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2c5:0x0]: losing layout lock 00010000:00010000:0.0:1775764770.723618:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421bf1f lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c5:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f81f3 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764770.725117:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2c5:0x0]: losing layout lock 00010000:00010000:0.0:1775764770.725174:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4600/0x36f9bdc9e421c308 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2c5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f8eb1 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764770.725505:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421bf1f lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2c5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f81f3 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764770.725512:0:19724:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1775764770.725515:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4600/0x36f9bdc9e421c308 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2c5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f8eb1 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764770.725536:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421bf1f lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2c5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f81f3 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000020:00100000:0.0:1775764770.725563:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 2 is allocated opc 36, max 7 00000100:00100000:0.0:1775764770.725578:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e4380 x1862022966919680/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764770.725605:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4380 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966919680:192.168.202.146@tcp:36:dbench.0 00000100:00100000:0.0:1775764770.725806:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f180 going to sleep for 11 seconds 00000100:00100000:3.0:1775764770.725984:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966919424:192.168.202.146@tcp:36:dbench.0 00000080:00200000:3.0:1775764770.726037:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x296:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1775764770.726046:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764770.726060:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:3.0:1775764770.726061:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:3.0:1775764770.726083:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x296:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764770.729031:0:19723:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x296:0x0](ffff8f9952dcdb88) 00000008:00010000:3.0:1775764770.729088:0:19723:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 515 [cma]time 1775764732 1775764732 0 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f22973fd/0x36f9bdc9e41ca5ec lrc: 1/0,0 mode: PW/PW res: [0x280000403:0x44:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eda44 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 93 lru_type: 1 00000080:00200000:3.0:1775764770.729162:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.729167:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.729171:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:3.0:1775764770.729176:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.729178:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:3.0:1775764770.729180:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.729182:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:3.0:1775764770.729184:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.729186:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=COREL, flags=17 00000080:00200000:3.0:1775764770.729188:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x291:0x0](ffff8f9952dccb08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.729197:0:19723:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=ROSE.CPT, dir=[0x200000401:0x291:0x0](ffff8f9952dccb08), flags=1 00000080:00200000:3.0:1775764770.729200:0:19723:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=ROSE.CPT, dir=[0x200000401:0x291:0x0](ffff8f9952dccb08), intent=getattr 00010000:00010000:3.0:1775764770.729214:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080400/0x36f9bdc9e421c39b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f91c8 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:3.0:1775764770.729225:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c39b from it ffffafbac1c7b900 00010000:00010000:3.0:1775764770.729228:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080400/0x36f9bdc9e421c39b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f91c8 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764770.729234:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080400/0x36f9bdc9e421c39b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f91c8 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:3.0:1775764770.729249:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file ROSE.CPT:[0x200000401:0x2a6:0x0](ffff8f9952d10908), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764770.729253:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2a6:0x0](ffff8f9952d10908),name=ROSE.CPT 00000002:00010000:3.0:1775764770.729266:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2a6:0x0]) in obj [0x200000401:0x291:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764770.729272:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081e00/0x36f9bdc9e4217b41 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2a6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e9fd8 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764770.729280:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081e00/0x36f9bdc9e4217b41 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2a6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e9fd8 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764770.729284:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x291:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764770.729289:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2a6:0x0](ffff8f9952d10908) 00000080:00010000:3.0:1775764770.729291:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2a6:0x0](ffff8f9952d10908) for lock 0x36f9bdc9e4217b41 00000080:00010000:3.0:1775764770.729295:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e4217b41 from it ffffafbac1c7bcb8 00010000:00010000:3.0:1775764770.729297:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081e00/0x36f9bdc9e4217b41 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5e9fd8 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764770.729301:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081e00/0x36f9bdc9e4217b41 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e9fd8 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764770.729317:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2a6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764770.729327:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2a6:0x0] 00000008:00010000:3.0:1775764770.729349:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1775764770.729351:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d5cb0, osc lock ffff8f9966acef70, flags 0x400001002 00010000:00010000:3.0:1775764770.729364:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000e02763ec/0x36f9bdc9e41ca82a lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x4c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4edfa7 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 1 00010000:00010000:3.0:1775764770.729369:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000e02763ec/0x36f9bdc9e41ca82a lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x4c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4edfa7 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 1 00000020:00200000:3.0:1775764770.729377:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1775764770.729379:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000e02763ec/0x36f9bdc9e41ca82a lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x4c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4edfa7 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 1 00010000:00010000:3.0:1775764770.729385:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000e02763ec/0x36f9bdc9e41ca82a lrc: 4/0,0 mode: PW/PW res: [0x280000403:0x4c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4edfa7 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 1 00000020:00200000:3.0:1775764770.729394:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1775764770.729402:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2a6:0x0] updating i_size 260096 i_blocks 1 00000020:00010000:3.0:1775764770.729405:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef50db0 (ffff8f99703141b8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764770.729408:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef50db0 (ffff8f99703141b8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.729411:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d5cb0 (ffff8f9970314128/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.729414:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef50db0 (ffff8f99703141b8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764770.729416:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d5cb0 (ffff8f99703141b8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764770.729433:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2a6:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764770.729453:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.729456:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.729459:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:3.0:1775764770.729461:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.729462:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=16 00000080:00200000:3.0:1775764770.729464:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.729465:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:3.0:1775764770.729466:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.729468:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=COREL, flags=16 00000080:00200000:3.0:1775764770.729470:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x291:0x0](ffff8f9952dccb08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.729473:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=ROSE.CPT, flags=0 00000080:00200000:3.0:1775764770.729475:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x291:0x0](ffff8f9952dccb08), inode mode 41c0 mask 3 00000080:00200000:3.0:1775764770.729479:0:19723:0:(namei.c:2395:ll_unlink()) VFS Op:name=ROSE.CPT, dir=[0x200000401:0x291:0x0](ffff8f9952dccb08) 00010000:00010000:3.0:1775764770.729507:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080400/0x36f9bdc9e421c39b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f91c8 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764770.729522:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2a6:0x0]: losing layout lock 00010000:00010000:3.0:1775764770.729537:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081e00/0x36f9bdc9e4217b41 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5e9fd8 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764770.729545:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2a6:0x0]: losing layout lock 00010000:00010000:3.0:1775764770.729578:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080400/0x36f9bdc9e421c39b lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2a6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f91c8 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764770.729583:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081e00/0x36f9bdc9e4217b41 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2a6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5e9fd8 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764770.729588:0:19723:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1775764770.729590:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080400/0x36f9bdc9e421c39b lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2a6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f91c8 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764770.729597:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081e00/0x36f9bdc9e4217b41 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2a6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5e9fd8 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000020:00100000:3.0:1775764770.729615:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1775764770.729638:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970edd880 x1862022966919936/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764770.729663:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966919936:192.168.202.146@tcp:36:dbench.0 00000100:00100000:3.0:1775764770.729775:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99707ef6c0 going to sleep for 11 seconds 00000100:00100000:3.0:1775764770.731170:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4380 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966919680:192.168.202.146@tcp:36:dbench.0 00000080:00200000:3.0:1775764770.731225:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.731231:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.731235:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:3.0:1775764770.731239:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.731242:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:3.0:1775764770.731245:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.731247:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:3.0:1775764770.731250:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.731253:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:3.0:1775764770.731255:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.731265:0:19724:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=COURSES.VAL, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), flags=1 00000080:00200000:3.0:1775764770.731269:0:19724:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=COURSES.VAL, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), intent=getattr 00010000:00010000:3.0:1775764770.731286:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4a00/0x36f9bdc9e421c30f lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xeba:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8ed4 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:3.0:1775764770.731298:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c30f from it ffffafbac1c5b900 00010000:00010000:3.0:1775764770.731301:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4a00/0x36f9bdc9e421c30f lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xeba:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8ed4 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764770.731309:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4a00/0x36f9bdc9e421c30f lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xeba:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8ed4 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:3.0:1775764770.731331:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file COURSES.VAL:[0x200000401:0xeba:0x0](ffff8f9952eedb88), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764770.731336:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xeba:0x0](ffff8f9952eedb88),name=COURSES.VAL 00000002:00010000:3.0:1775764770.731349:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xeba:0x0]) in obj [0x200000401:0x2b8:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764770.731359:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093dc00/0x36f9bdc9e421ba4f lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xeba:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f7328 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764770.731369:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093dc00/0x36f9bdc9e421ba4f lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xeba:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f7328 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764770.731377:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2b8:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764770.731381:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xeba:0x0](ffff8f9952eedb88) 00000080:00010000:3.0:1775764770.731383:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xeba:0x0](ffff8f9952eedb88) for lock 0x36f9bdc9e421ba4f 00000080:00010000:3.0:1775764770.731388:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421ba4f from it ffffafbac1c5bcb8 00010000:00010000:3.0:1775764770.731390:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093dc00/0x36f9bdc9e421ba4f lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xeba:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f7328 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764770.731396:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093dc00/0x36f9bdc9e421ba4f lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xeba:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f7328 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764770.731436:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xeba:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764770.731445:0:19724:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xeba:0x0] 00000008:00010000:3.0:1775764770.731467:0:19724:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1775764770.731470:0:19724:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d5cb0, osc lock ffff8f9966acef70, flags 0x400001002 00010000:00010000:3.0:1775764770.731484:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000839c629c/0x36f9bdc9e421a3c0 lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x651:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f2e5c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 0 00010000:00010000:3.0:1775764770.731494:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000839c629c/0x36f9bdc9e421a3c0 lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x651:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f2e5c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 0 00000020:00200000:3.0:1775764770.731504:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1775764770.731506:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000839c629c/0x36f9bdc9e421a3c0 lrc: 6/0,1 mode: PW/PW res: [0x280000403:0x651:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5f2e5c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 0 00010000:00010000:3.0:1775764770.731512:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000839c629c/0x36f9bdc9e421a3c0 lrc: 5/0,0 mode: PW/PW res: [0x280000403:0x651:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f2e5c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 0 00000020:00200000:3.0:1775764770.731521:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1775764770.731529:0:19724:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0xeba:0x0] updating i_size 543 i_blocks 1 00000020:00010000:3.0:1775764770.731533:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef50db0 (ffff8f99703141b8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764770.731536:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef50db0 (ffff8f99703141b8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.731539:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d5cb0 (ffff8f9970314128/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.731543:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef50db0 (ffff8f99703141b8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764770.731545:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d5cb0 (ffff8f99703141b8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764770.731565:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xeba:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764770.731585:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.731589:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.731591:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:3.0:1775764770.731595:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.731597:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=16 00000080:00200000:3.0:1775764770.731599:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.731602:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:3.0:1775764770.731604:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.731606:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=16 00000080:00200000:3.0:1775764770.731608:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.731613:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=COURSES.VAL, flags=0 00000080:00200000:3.0:1775764770.731616:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 3 00000080:00200000:3.0:1775764770.731632:0:19724:0:(namei.c:2395:ll_unlink()) VFS Op:name=COURSES.VAL, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8) 00010000:00010000:3.0:1775764770.731668:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4a00/0x36f9bdc9e421c30f lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xeba:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f8ed4 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764770.731689:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0xeba:0x0]: losing layout lock 00010000:00010000:3.0:1775764770.731707:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093dc00/0x36f9bdc9e421ba4f lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xeba:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f7328 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764770.731723:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0xeba:0x0]: losing layout lock 00010000:00010000:3.0:1775764770.731806:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4a00/0x36f9bdc9e421c30f lrc: 1/0,0 mode: --/PR res: [0x200000401:0xeba:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f8ed4 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764770.731815:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093dc00/0x36f9bdc9e421ba4f lrc: 1/0,0 mode: --/PR res: [0x200000401:0xeba:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f7328 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764770.731820:0:19724:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1775764770.731823:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4a00/0x36f9bdc9e421c30f lrc: 0/0,0 mode: --/PR res: [0x200000401:0xeba:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f8ed4 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764770.731841:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093dc00/0x36f9bdc9e421ba4f lrc: 0/0,0 mode: --/PR res: [0x200000401:0xeba:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f7328 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000020:00100000:3.0:1775764770.731860:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 2 is allocated opc 36, max 7 00000100:00100000:3.0:1775764770.731873:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970edfb80 x1862022966920192/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764770.731892:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966920192:192.168.202.146@tcp:36:dbench.0 00000100:00100000:3.0:1775764770.731950:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99707ef900 going to sleep for 11 seconds 00000100:00100000:3.0:1775764770.739834:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966919936:192.168.202.146@tcp:36:dbench.0 00000080:00200000:3.0:1775764770.739926:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2a6:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1775764770.739940:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764770.739960:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:3.0:1775764770.739962:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:3.0:1775764770.739999:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2a6:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764770.740410:0:19723:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2a6:0x0](ffff8f9952d10908) 00000008:00010000:3.0:1775764770.740455:0:19723:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 1 [cma]time 1775764054 1775764054 0 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000e02763ec/0x36f9bdc9e41ca82a lrc: 1/0,0 mode: PW/PW res: [0x280000403:0x4c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4edfa7 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 1 00000080:00200000:3.0:1775764770.740561:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.740568:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.740575:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:3.0:1775764770.740581:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.740585:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:3.0:1775764770.740588:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.740592:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:3.0:1775764770.740594:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.740597:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=COREL, flags=17 00000080:00200000:3.0:1775764770.740599:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x291:0x0](ffff8f9952dccb08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.740612:0:19723:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=TRAIN.CDR, dir=[0x200000401:0x291:0x0](ffff8f9952dccb08), flags=1 00000080:00200000:3.0:1775764770.740618:0:19723:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=TRAIN.CDR, dir=[0x200000401:0x291:0x0](ffff8f9952dccb08), intent=getattr 00010000:00010000:3.0:1775764770.740651:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080800/0x36f9bdc9e421c3a2 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f91e4 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:3.0:1775764770.740665:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c3a2 from it ffffafbac1c7b900 00010000:00010000:3.0:1775764770.740668:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080800/0x36f9bdc9e421c3a2 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f91e4 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764770.740677:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080800/0x36f9bdc9e421c3a2 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f91e4 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:3.0:1775764770.740699:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file TRAIN.CDR:[0x200000401:0x2a2:0x0](ffff8f9952dcf448), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764770.740705:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2a2:0x0](ffff8f9952dcf448),name=TRAIN.CDR 00000002:00010000:3.0:1775764770.740721:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2a2:0x0]) in obj [0x200000401:0x291:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764770.740731:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b8a00/0x36f9bdc9e421787e lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2a2:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e96d2 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764770.740772:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b8a00/0x36f9bdc9e421787e lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2a2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e96d2 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764770.740781:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x291:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764770.740786:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2a2:0x0](ffff8f9952dcf448) 00000080:00010000:3.0:1775764770.740789:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2a2:0x0](ffff8f9952dcf448) for lock 0x36f9bdc9e421787e 00000080:00010000:3.0:1775764770.740794:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421787e from it ffffafbac1c7bcb8 00010000:00010000:3.0:1775764770.740797:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b8a00/0x36f9bdc9e421787e lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5e96d2 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764770.740803:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b8a00/0x36f9bdc9e421787e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e96d2 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764770.740827:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2a2:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764770.740834:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2a2:0x0] 00000008:00010000:3.0:1775764770.740847:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1775764770.740849:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d5cb0, osc lock ffff8f9966acef70, flags 0x400001002 00010000:00010000:3.0:1775764770.740863:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000063e812bc/0x36f9bdc9e4210eba lrc: 2/1,0 mode: PR/PR res: [0x280000403:0x4a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5d3d47 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 1 00010000:00010000:3.0:1775764770.740870:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000063e812bc/0x36f9bdc9e4210eba lrc: 2/1,0 mode: PR/PR res: [0x280000403:0x4a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5d3d47 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 1 00000020:00200000:3.0:1775764770.740878:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1775764770.740880:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000063e812bc/0x36f9bdc9e4210eba lrc: 5/1,0 mode: PR/PR res: [0x280000403:0x4a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5d3d47 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 1 00010000:00010000:3.0:1775764770.740885:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000063e812bc/0x36f9bdc9e4210eba lrc: 4/0,0 mode: PR/PR res: [0x280000403:0x4a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5d3d47 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00000020:00200000:3.0:1775764770.740892:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1775764770.740898:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2a2:0x0] updating i_size 260096 i_blocks 515 00000020:00010000:3.0:1775764770.740900:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef50db0 (ffff8f99703141b8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764770.740902:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef50db0 (ffff8f99703141b8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.740904:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d5cb0 (ffff8f9970314128/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.740907:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef50db0 (ffff8f99703141b8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764770.740908:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d5cb0 (ffff8f99703141b8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764770.740918:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2a2:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764770.740931:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.740933:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.740936:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:3.0:1775764770.740938:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.740939:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=16 00000080:00200000:3.0:1775764770.740941:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.740942:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:3.0:1775764770.740944:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.740945:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=COREL, flags=16 00000080:00200000:3.0:1775764770.740947:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x291:0x0](ffff8f9952dccb08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.740950:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=TRAIN.CDR, flags=0 00000080:00200000:3.0:1775764770.740953:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x291:0x0](ffff8f9952dccb08), inode mode 41c0 mask 3 00000080:00200000:3.0:1775764770.740956:0:19723:0:(namei.c:2395:ll_unlink()) VFS Op:name=TRAIN.CDR, dir=[0x200000401:0x291:0x0](ffff8f9952dccb08) 00010000:00010000:3.0:1775764770.740982:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080800/0x36f9bdc9e421c3a2 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a2:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f91e4 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764770.740996:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2a2:0x0]: losing layout lock 00010000:00010000:3.0:1775764770.741008:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b8a00/0x36f9bdc9e421787e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a2:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5e96d2 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764770.741017:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2a2:0x0]: losing layout lock 00010000:00010000:3.0:1775764770.741042:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080800/0x36f9bdc9e421c3a2 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2a2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f91e4 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764770.741047:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b8a00/0x36f9bdc9e421787e lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2a2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5e96d2 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764770.741051:0:19723:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1775764770.741053:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080800/0x36f9bdc9e421c3a2 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2a2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f91e4 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764770.741061:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b8a00/0x36f9bdc9e421787e lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2a2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5e96d2 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000020:00100000:3.0:1775764770.741076:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1775764770.741083:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970edf800 x1862022966920448/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764770.741101:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966920448:192.168.202.146@tcp:36:dbench.0 00000100:00100000:3.0:1775764770.741352:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99707ef6c0 going to sleep for 11 seconds 00000100:00100000:3.0:1775764770.743563:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966920192:192.168.202.146@tcp:36:dbench.0 00000080:00200000:3.0:1775764770.744059:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xeba:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1775764770.744072:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764770.744090:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:3.0:1775764770.744091:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000008:00010000:3.0:1775764770.744110:0:19724:0:(osc_cache.c:1167:osc_extent_make_ready()) ### extent: 000000000a10ed43 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000839c629c/0x36f9bdc9e421a3c0 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x651:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f2e5c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 0 00000008:00010000:3.0:1775764770.744141:0:19724:0:(osc_cache.c:901:osc_extent_finish()) ### extent: 000000000a10ed43 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000839c629c/0x36f9bdc9e421a3c0 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x651:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f2e5c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 0 00000008:00010000:3.0:1775764770.744164:0:19724:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000a10ed43 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000839c629c/0x36f9bdc9e421a3c0 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x651:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f2e5c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 0 00000080:00200000:3.0:1775764770.744209:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xeba:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764770.744260:0:19724:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0xeba:0x0](ffff8f9952eedb88) 00000008:00010000:3.0:1775764770.744292:0:19724:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 543 blocks 1 [cma]time 1775764754 1775764754 0 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000839c629c/0x36f9bdc9e421a3c0 lrc: 1/0,0 mode: PW/PW res: [0x280000403:0x651:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f2e5c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 0 00000080:00200000:3.0:1775764770.744367:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.744371:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.744375:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:3.0:1775764770.744379:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.744381:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:3.0:1775764770.744383:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.744385:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:3.0:1775764770.744387:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.744389:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:3.0:1775764770.744391:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.744400:0:19724:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=STUDENTS.YG0, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), flags=1 00000080:00200000:3.0:1775764770.744403:0:19724:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=STUDENTS.YG0, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), intent=getattr 00010000:00010000:3.0:1775764770.744417:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4800/0x36f9bdc9e421c316 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8efe expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:3.0:1775764770.744427:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c316 from it ffffafbac1c5b900 00010000:00010000:3.0:1775764770.744430:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4800/0x36f9bdc9e421c316 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8efe expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764770.744436:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4800/0x36f9bdc9e421c316 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8efe expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:3.0:1775764770.744455:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file STUDENTS.YG0:[0x200000401:0x2c7:0x0](ffff8f9952d97448), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764770.744459:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2c7:0x0](ffff8f9952d97448),name=STUDENTS.YG0 00000002:00010000:3.0:1775764770.744472:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2c7:0x0]) in obj [0x200000401:0x2b8:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764770.744479:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421bf49 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2c7:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8271 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764770.744487:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421bf49 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2c7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8271 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764770.744491:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2b8:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764770.744495:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2c7:0x0](ffff8f9952d97448) 00000080:00010000:3.0:1775764770.744497:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2c7:0x0](ffff8f9952d97448) for lock 0x36f9bdc9e421bf49 00000080:00010000:3.0:1775764770.744501:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421bf49 from it ffffafbac1c5bcb8 00010000:00010000:3.0:1775764770.744503:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421bf49 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8271 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764770.744508:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421bf49 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8271 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764770.744523:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2c7:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764770.744527:0:19724:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2c7:0x0] 00000008:00010000:3.0:1775764770.744540:0:19724:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1775764770.744543:0:19724:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d5cb0, osc lock ffff8f9966acef70, flags 0x400001002 00010000:00010000:3.0:1775764770.744554:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001fc9e7d4/0x36f9bdc9e41cacc9 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x5a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eebfc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 47 lru_type: 1 00010000:00010000:3.0:1775764770.744565:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001fc9e7d4/0x36f9bdc9e41cacc9 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x5a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eebfc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 47 lru_type: 1 00000020:00200000:3.0:1775764770.744573:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1775764770.744576:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001fc9e7d4/0x36f9bdc9e41cacc9 lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x5a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4eebfc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 47 lru_type: 1 00010000:00010000:3.0:1775764770.744580:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001fc9e7d4/0x36f9bdc9e41cacc9 lrc: 4/0,0 mode: PW/PW res: [0x240000403:0x5a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eebfc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 48 lru_type: 1 00000020:00200000:3.0:1775764770.744586:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1775764770.744591:0:19724:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2c7:0x0] updating i_size 260096 i_blocks 1 00000020:00010000:3.0:1775764770.744593:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef50db0 (ffff8f99703141b8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764770.744595:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef50db0 (ffff8f99703141b8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.744597:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d5cb0 (ffff8f9970314128/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.744600:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef50db0 (ffff8f99703141b8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764770.744601:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d5cb0 (ffff8f99703141b8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764770.745183:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2c7:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764770.745201:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.745204:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.745207:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:3.0:1775764770.745210:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.745211:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=16 00000080:00200000:3.0:1775764770.745213:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.745214:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:3.0:1775764770.745216:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.745218:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=16 00000080:00200000:3.0:1775764770.745219:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.745223:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=STUDENTS.YG0, flags=0 00000080:00200000:3.0:1775764770.745225:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 3 00000080:00200000:3.0:1775764770.745228:0:19724:0:(namei.c:2395:ll_unlink()) VFS Op:name=STUDENTS.YG0, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8) 00010000:00010000:3.0:1775764770.745260:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4800/0x36f9bdc9e421c316 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c7:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f8efe expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764770.745274:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2c7:0x0]: losing layout lock 00010000:00010000:3.0:1775764770.745288:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421bf49 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c7:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f8271 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764770.745296:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2c7:0x0]: losing layout lock 00010000:00010000:3.0:1775764770.745333:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4800/0x36f9bdc9e421c316 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2c7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f8efe expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764770.745339:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421bf49 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2c7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f8271 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764770.745342:0:19724:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1775764770.745344:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4800/0x36f9bdc9e421c316 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2c7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f8efe expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764770.745353:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421bf49 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2c7:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f8271 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000020:00100000:3.0:1775764770.745368:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 2 is allocated opc 36, max 7 00000100:00100000:3.0:1775764770.745376:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970eddc00 x1862022966920704/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764770.745396:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966920704:192.168.202.146@tcp:36:dbench.0 00000100:00100000:3.0:1775764770.745588:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99707ef900 going to sleep for 11 seconds 00000100:00100000:3.0:1775764770.746282:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966920448:192.168.202.146@tcp:36:dbench.0 00000080:00200000:3.0:1775764770.746339:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2a2:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1775764770.746348:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764770.746363:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:3.0:1775764770.746364:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:3.0:1775764770.746384:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2a2:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764770.746720:0:19723:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2a2:0x0](ffff8f9952dcf448) 00000008:00010000:3.0:1775764770.746835:0:19723:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 260096 blocks 515 [cma]time: 1775764054 1775764054 0 by oinfo size 260096 blocks 515 [cma]time 1775764054 1775764054 1775764677 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000063e812bc/0x36f9bdc9e4210eba lrc: 1/0,0 mode: PR/PR res: [0x280000403:0x4a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5d3d47 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00000080:00200000:3.0:1775764770.746897:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.746901:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.746910:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:3.0:1775764770.746914:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.746916:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:3.0:1775764770.746919:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.746921:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:3.0:1775764770.746923:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.746925:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=COREL, flags=17 00000080:00200000:3.0:1775764770.746927:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x291:0x0](ffff8f9952dccb08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.746936:0:19723:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=GRAPHIC2.CDR, dir=[0x200000401:0x291:0x0](ffff8f9952dccb08), flags=1 00000080:00200000:3.0:1775764770.746938:0:19723:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=GRAPHIC2.CDR, dir=[0x200000401:0x291:0x0](ffff8f9952dccb08), intent=getattr 00010000:00010000:3.0:1775764770.746951:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081600/0x36f9bdc9e421c3a9 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9200 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:3.0:1775764770.746965:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c3a9 from it ffffafbac1c7b900 00010000:00010000:3.0:1775764770.746967:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081600/0x36f9bdc9e421c3a9 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9200 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764770.746973:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081600/0x36f9bdc9e421c3a9 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9200 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:3.0:1775764770.746988:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file GRAPHIC2.CDR:[0x200000401:0x2a4:0x0](ffff8f9952d88908), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764770.746992:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2a4:0x0](ffff8f9952d88908),name=GRAPHIC2.CDR 00000002:00010000:3.0:1775764770.747002:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2a4:0x0]) in obj [0x200000401:0x291:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764770.747008:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99413d2800/0x36f9bdc9e4217a45 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2a4:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e9ca5 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764770.747014:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99413d2800/0x36f9bdc9e4217a45 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2a4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e9ca5 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764770.747019:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x291:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764770.747022:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2a4:0x0](ffff8f9952d88908) 00000080:00010000:3.0:1775764770.747024:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2a4:0x0](ffff8f9952d88908) for lock 0x36f9bdc9e4217a45 00000080:00010000:3.0:1775764770.747027:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e4217a45 from it ffffafbac1c7bcb8 00010000:00010000:3.0:1775764770.747029:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99413d2800/0x36f9bdc9e4217a45 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5e9ca5 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764770.747034:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99413d2800/0x36f9bdc9e4217a45 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e9ca5 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764770.747052:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2a4:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764770.747057:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2a4:0x0] 00000008:00010000:3.0:1775764770.747071:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1775764770.747073:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d5cb0, osc lock ffff8f9966acef70, flags 0x400001002 00010000:00010000:3.0:1775764770.747086:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000072431b3a/0x36f9bdc9e421106c lrc: 2/1,0 mode: PR/PR res: [0x280000403:0x4b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5d438a expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 1 00010000:00010000:3.0:1775764770.747092:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000072431b3a/0x36f9bdc9e421106c lrc: 2/1,0 mode: PR/PR res: [0x280000403:0x4b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5d438a expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 1 00000020:00200000:3.0:1775764770.747101:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1775764770.747103:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000072431b3a/0x36f9bdc9e421106c lrc: 5/1,0 mode: PR/PR res: [0x280000403:0x4b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5d438a expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 1 00010000:00010000:3.0:1775764770.747107:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000072431b3a/0x36f9bdc9e421106c lrc: 4/0,0 mode: PR/PR res: [0x280000403:0x4b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5d438a expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 1 00000020:00200000:3.0:1775764770.747113:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1775764770.747118:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2a4:0x0] updating i_size 260096 i_blocks 515 00000020:00010000:3.0:1775764770.747121:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef50db0 (ffff8f99703141b8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764770.747122:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef50db0 (ffff8f99703141b8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.747124:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d5cb0 (ffff8f9970314128/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.747127:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef50db0 (ffff8f99703141b8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764770.747128:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d5cb0 (ffff8f99703141b8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764770.747138:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2a4:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764770.747149:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.747152:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.747154:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:3.0:1775764770.747155:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.747157:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=16 00000080:00200000:3.0:1775764770.747158:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.747160:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:3.0:1775764770.747162:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.747163:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=COREL, flags=16 00000080:00200000:3.0:1775764770.747164:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x291:0x0](ffff8f9952dccb08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.747168:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=GRAPHIC2.CDR, flags=0 00000080:00200000:3.0:1775764770.747170:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x291:0x0](ffff8f9952dccb08), inode mode 41c0 mask 3 00000080:00200000:3.0:1775764770.747173:0:19723:0:(namei.c:2395:ll_unlink()) VFS Op:name=GRAPHIC2.CDR, dir=[0x200000401:0x291:0x0](ffff8f9952dccb08) 00010000:00010000:3.0:1775764770.747195:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081600/0x36f9bdc9e421c3a9 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a4:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f9200 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764770.747207:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2a4:0x0]: losing layout lock 00010000:00010000:3.0:1775764770.747218:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99413d2800/0x36f9bdc9e4217a45 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a4:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5e9ca5 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764770.747226:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2a4:0x0]: losing layout lock 00010000:00010000:3.0:1775764770.747245:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081600/0x36f9bdc9e421c3a9 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2a4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f9200 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764770.747250:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99413d2800/0x36f9bdc9e4217a45 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2a4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5e9ca5 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764770.747253:0:19723:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1775764770.747255:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081600/0x36f9bdc9e421c3a9 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2a4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f9200 expref: -99 pid: 19863 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764770.747265:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99413d2800/0x36f9bdc9e4217a45 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2a4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5e9ca5 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000020:00100000:3.0:1775764770.747278:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1775764770.747283:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970edc000 x1862022966920960/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764770.747296:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966920960:192.168.202.146@tcp:36:dbench.0 00000100:00100000:3.0:1775764770.747345:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99707ef6c0 going to sleep for 11 seconds 00000100:00100000:3.0:1775764770.751573:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966920704:192.168.202.146@tcp:36:dbench.0 00000080:00200000:3.0:1775764770.751669:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.751676:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.751682:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:3.0:1775764770.751689:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.751693:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:3.0:1775764770.751696:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.751699:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:3.0:1775764770.751702:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.751705:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:3.0:1775764770.751708:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.751730:0:19724:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=ANSWER.MB, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), flags=1 00000080:00200000:3.0:1775764770.751734:0:19724:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=ANSWER.MB, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), intent=getattr 00010000:00010000:3.0:1775764770.751781:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5000/0x36f9bdc9e421c31d lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xec8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8f21 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:3.0:1775764770.751795:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c31d from it ffffafbac1c5b900 00010000:00010000:3.0:1775764770.751798:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5000/0x36f9bdc9e421c31d lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xec8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8f21 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764770.751804:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5000/0x36f9bdc9e421c31d lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xec8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8f21 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:3.0:1775764770.751822:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file ANSWER.MB:[0x200000401:0xec8:0x0](ffff8f9952eb42c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764770.751826:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xec8:0x0](ffff8f9952eb42c8),name=ANSWER.MB 00000002:00010000:3.0:1775764770.751839:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xec8:0x0]) in obj [0x200000401:0x2b8:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764770.751849:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf800/0x36f9bdc9e421aea2 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xec8:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f500c expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764770.751857:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf800/0x36f9bdc9e421aea2 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xec8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f500c expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764770.751862:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2b8:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764770.751865:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xec8:0x0](ffff8f9952eb42c8) 00000080:00010000:3.0:1775764770.751867:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xec8:0x0](ffff8f9952eb42c8) for lock 0x36f9bdc9e421aea2 00000080:00010000:3.0:1775764770.751870:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421aea2 from it ffffafbac1c5bcb8 00010000:00010000:3.0:1775764770.751872:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf800/0x36f9bdc9e421aea2 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xec8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f500c expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764770.751876:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf800/0x36f9bdc9e421aea2 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xec8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f500c expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764770.751912:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xec8:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764770.751921:0:19724:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xec8:0x0] 00000008:00010000:3.0:1775764770.751946:0:19724:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1775764770.751950:0:19724:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d5cb0, osc lock ffff8f9966acef70, flags 0x400001002 00010000:00010000:3.0:1775764770.751966:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000030aeec1d/0x36f9bdc9e421ad75 lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x658:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f4c54 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 0 00010000:00010000:3.0:1775764770.751977:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000030aeec1d/0x36f9bdc9e421ad75 lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x658:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f4c54 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 0 00000020:00200000:3.0:1775764770.751989:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1775764770.751993:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000030aeec1d/0x36f9bdc9e421ad75 lrc: 6/0,1 mode: PW/PW res: [0x280000403:0x658:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5f4c54 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 0 00010000:00010000:3.0:1775764770.752000:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000030aeec1d/0x36f9bdc9e421ad75 lrc: 5/0,0 mode: PW/PW res: [0x280000403:0x658:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f4c54 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 0 00000020:00200000:3.0:1775764770.752010:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1775764770.752017:0:19724:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0xec8:0x0] updating i_size 4096 i_blocks 1 00000020:00010000:3.0:1775764770.752020:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef50db0 (ffff8f99703141b8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764770.752022:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef50db0 (ffff8f99703141b8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.752024:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d5cb0 (ffff8f9970314128/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.752027:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef50db0 (ffff8f99703141b8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764770.752028:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d5cb0 (ffff8f99703141b8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764770.752048:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xec8:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764770.752064:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.752066:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.752068:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:3.0:1775764770.752070:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.752071:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=16 00000080:00200000:3.0:1775764770.752072:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.752074:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:3.0:1775764770.752076:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.752077:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=16 00000080:00200000:3.0:1775764770.752079:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.752082:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=ANSWER.MB, flags=0 00000080:00200000:3.0:1775764770.752085:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 3 00000080:00200000:3.0:1775764770.752088:0:19724:0:(namei.c:2395:ll_unlink()) VFS Op:name=ANSWER.MB, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8) 00010000:00010000:3.0:1775764770.752127:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5000/0x36f9bdc9e421c31d lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xec8:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f8f21 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764770.752148:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0xec8:0x0]: losing layout lock 00010000:00010000:3.0:1775764770.752162:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf800/0x36f9bdc9e421aea2 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xec8:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f500c expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764770.752171:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0xec8:0x0]: losing layout lock 00010000:00010000:3.0:1775764770.752194:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5000/0x36f9bdc9e421c31d lrc: 1/0,0 mode: --/PR res: [0x200000401:0xec8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f8f21 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764770.752198:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf800/0x36f9bdc9e421aea2 lrc: 1/0,0 mode: --/PR res: [0x200000401:0xec8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f500c expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764770.752202:0:19724:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1775764770.752204:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5000/0x36f9bdc9e421c31d lrc: 0/0,0 mode: --/PR res: [0x200000401:0xec8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f8f21 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764770.752213:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf800/0x36f9bdc9e421aea2 lrc: 0/0,0 mode: --/PR res: [0x200000401:0xec8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f500c expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000020:00100000:3.0:1775764770.752227:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 2 is allocated opc 36, max 7 00000100:00100000:3.0:1775764770.752234:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970eded80 x1862022966921216/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764770.752256:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966921216:192.168.202.146@tcp:36:dbench.0 00000100:00100000:3.0:1775764770.752309:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99707ef900 going to sleep for 11 seconds 00000100:00100000:3.0:1775764770.760209:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966921216:192.168.202.146@tcp:36:dbench.0 00000080:00200000:3.0:1775764770.760264:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xec8:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1775764770.760274:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764770.760288:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:3.0:1775764770.760289:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000008:00010000:3.0:1775764770.760303:0:19724:0:(osc_cache.c:1167:osc_extent_make_ready()) ### extent: 0000000089b3b2d3 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000030aeec1d/0x36f9bdc9e421ad75 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x658:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f4c54 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 0 00000008:00010000:3.0:1775764770.760329:0:19724:0:(osc_cache.c:901:osc_extent_finish()) ### extent: 0000000089b3b2d3 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000030aeec1d/0x36f9bdc9e421ad75 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x658:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f4c54 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 0 00000008:00010000:3.0:1775764770.760346:0:19724:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 0000000089b3b2d3 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000030aeec1d/0x36f9bdc9e421ad75 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x658:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f4c54 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 0 00000080:00200000:3.0:1775764770.760369:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xec8:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764770.760413:0:19724:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0xec8:0x0](ffff8f9952eb42c8) 00000008:00010000:3.0:1775764770.760446:0:19724:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 4096 blocks 1 [cma]time 1775764760 1775764760 0 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000030aeec1d/0x36f9bdc9e421ad75 lrc: 1/0,0 mode: PW/PW res: [0x280000403:0x658:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f4c54 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 0 00000080:00200000:3.0:1775764770.760512:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.760516:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.760519:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:3.0:1775764770.760522:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.760525:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:3.0:1775764770.760527:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.760529:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:3.0:1775764770.760531:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.760533:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:3.0:1775764770.760535:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.760546:0:19724:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=STUDENTS.DB, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), flags=1 00000080:00200000:3.0:1775764770.760549:0:19724:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=STUDENTS.DB, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), intent=getattr 00010000:00010000:3.0:1775764770.760561:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4400/0x36f9bdc9e421c324 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8f3d expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:3.0:1775764770.760571:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c324 from it ffffafbac1c5b900 00010000:00010000:3.0:1775764770.760573:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4400/0x36f9bdc9e421c324 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8f3d expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764770.760579:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4400/0x36f9bdc9e421c324 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8f3d expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:3.0:1775764770.760597:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file STUDENTS.DB:[0x200000401:0x2c1:0x0](ffff8f9952e4d348), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764770.760600:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2c1:0x0](ffff8f9952e4d348),name=STUDENTS.DB 00000002:00010000:3.0:1775764770.760952:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2c1:0x0]) in obj [0x200000401:0x2b8:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764770.760964:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb000/0x36f9bdc9e421becb lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2c1:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f80f7 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764770.760975:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb000/0x36f9bdc9e421becb lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2c1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f80f7 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764770.760983:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2b8:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764770.760989:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2c1:0x0](ffff8f9952e4d348) 00000080:00010000:3.0:1775764770.760993:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2c1:0x0](ffff8f9952e4d348) for lock 0x36f9bdc9e421becb 00000080:00010000:3.0:1775764770.760997:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421becb from it ffffafbac1c5bcb8 00010000:00010000:3.0:1775764770.761000:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb000/0x36f9bdc9e421becb lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f80f7 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764770.761006:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb000/0x36f9bdc9e421becb lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f80f7 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764770.761026:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2c1:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764770.761030:0:19724:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2c1:0x0] 00000008:00010000:3.0:1775764770.761045:0:19724:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1775764770.761048:0:19724:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d5cb0, osc lock ffff8f9966acef70, flags 0x400001002 00010000:00010000:3.0:1775764770.761060:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f82f5f70/0x36f9bdc9e41cac05 lrc: 3/0,1 mode: PW/PW res: [0x240000403:0x57:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eea4a expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 50 lru_type: 1 00010000:00010000:3.0:1775764770.761067:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f82f5f70/0x36f9bdc9e41cac05 lrc: 3/0,1 mode: PW/PW res: [0x240000403:0x57:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eea4a expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 50 lru_type: 1 00000020:00200000:3.0:1775764770.761075:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1775764770.761077:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f82f5f70/0x36f9bdc9e41cac05 lrc: 6/0,1 mode: PW/PW res: [0x240000403:0x57:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4eea4a expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 50 lru_type: 1 00010000:00010000:3.0:1775764770.761081:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f82f5f70/0x36f9bdc9e41cac05 lrc: 5/0,0 mode: PW/PW res: [0x240000403:0x57:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eea4a expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 51 lru_type: 1 00000020:00200000:3.0:1775764770.761087:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1775764770.761093:0:19724:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2c1:0x0] updating i_size 260096 i_blocks 515 00000020:00010000:3.0:1775764770.761096:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef50db0 (ffff8f99703141b8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764770.761098:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef50db0 (ffff8f99703141b8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.761100:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d5cb0 (ffff8f9970314128/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.761103:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef50db0 (ffff8f99703141b8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764770.761104:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d5cb0 (ffff8f99703141b8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764770.761114:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2c1:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764770.761126:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.761128:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.761130:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:3.0:1775764770.761133:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.761134:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=16 00000080:00200000:3.0:1775764770.761136:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.761137:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:3.0:1775764770.761138:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.761140:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=16 00000080:00200000:3.0:1775764770.761141:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.761144:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=STUDENTS.DB, flags=0 00000080:00200000:3.0:1775764770.761146:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 3 00000080:00200000:3.0:1775764770.761149:0:19724:0:(namei.c:2395:ll_unlink()) VFS Op:name=STUDENTS.DB, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8) 00010000:00010000:3.0:1775764770.761170:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4400/0x36f9bdc9e421c324 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c1:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f8f3d expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764770.761183:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2c1:0x0]: losing layout lock 00010000:00010000:3.0:1775764770.761195:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb000/0x36f9bdc9e421becb lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c1:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f80f7 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764770.761203:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2c1:0x0]: losing layout lock 00010000:00010000:3.0:1775764770.761223:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4400/0x36f9bdc9e421c324 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2c1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f8f3d expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764770.761231:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb000/0x36f9bdc9e421becb lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2c1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f80f7 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764770.761235:0:19724:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1775764770.761237:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4400/0x36f9bdc9e421c324 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2c1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f8f3d expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764770.761244:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb000/0x36f9bdc9e421becb lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2c1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f80f7 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000020:00100000:3.0:1775764770.761257:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 2 is allocated opc 36, max 7 00000100:00100000:3.0:1775764770.761263:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970edd180 x1862022966921728/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764770.761277:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966921728:192.168.202.146@tcp:36:dbench.0 00000100:00100000:3.0:1775764770.761342:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99707ef900 going to sleep for 11 seconds 00000100:00100000:3.0:1775764770.764115:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966921728:192.168.202.146@tcp:36:dbench.0 00000080:00200000:3.0:1775764770.764235:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.764241:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.764248:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:3.0:1775764770.764254:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.764257:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:3.0:1775764770.764259:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.764263:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:3.0:1775764770.764265:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.764267:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:3.0:1775764770.764270:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.764273:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=COURSES.DB, flags=1 00010000:00010000:3.0:1775764770.764292:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941481000/0x36f9bdc9e421b9a0 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2ba:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f711b expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00010000:3.0:1775764770.764304:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421b9a0 from it ffffafbac1c5baf8 00010000:00010000:3.0:1775764770.764307:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941481000/0x36f9bdc9e421b9a0 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2ba:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f711b expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1775764770.764317:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941481000/0x36f9bdc9e421b9a0 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ba:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f711b expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 1 00000080:00200008:3.0:1775764770.764342:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file COURSES.DB:[0x200000401:0x2ba:0x0](ffff8f9952d8d348), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764770.764348:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2ba:0x0](ffff8f9952d8d348),name=COURSES.DB 00000002:00010000:3.0:1775764770.764368:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2ba:0x0]) in obj [0x200000401:0x2b8:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764770.764380:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941481000/0x36f9bdc9e421b9a0 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2ba:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f711b expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 1 00010000:00010000:3.0:1775764770.764396:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941481000/0x36f9bdc9e421b9a0 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2ba:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f711b expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 1 00000080:00200000:3.0:1775764770.764404:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2b8:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764770.764410:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2ba:0x0](ffff8f9952d8d348) 00000080:00010000:3.0:1775764770.764413:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2ba:0x0](ffff8f9952d8d348) for lock 0x36f9bdc9e421b9a0 00000080:00010000:3.0:1775764770.764417:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421b9a0 from it ffffafbac1c5bcb8 00010000:00010000:3.0:1775764770.764420:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941481000/0x36f9bdc9e421b9a0 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2ba:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f711b expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 1 00010000:00010000:3.0:1775764770.764431:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941481000/0x36f9bdc9e421b9a0 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ba:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f711b expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 1 00000080:00200000:3.0:1775764770.764461:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2ba:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764770.764467:0:19724:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2ba:0x0] 00000008:00010000:3.0:1775764770.764491:0:19724:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1775764770.764495:0:19724:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d5cb0, osc lock ffff8f9966acef70, flags 0x400001002 00010000:00010000:3.0:1775764770.764511:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000083f2c617/0x36f9bdc9e41caaf4 lrc: 3/0,1 mode: PW/PW res: [0x240000403:0x54:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee756 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 68 lru_type: 1 00010000:00010000:3.0:1775764770.764521:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000083f2c617/0x36f9bdc9e41caaf4 lrc: 3/0,1 mode: PW/PW res: [0x240000403:0x54:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee756 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 68 lru_type: 1 00000020:00200000:3.0:1775764770.764532:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1775764770.764534:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000083f2c617/0x36f9bdc9e41caaf4 lrc: 6/0,1 mode: PW/PW res: [0x240000403:0x54:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ee756 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 68 lru_type: 1 00010000:00010000:3.0:1775764770.764542:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000083f2c617/0x36f9bdc9e41caaf4 lrc: 5/0,0 mode: PW/PW res: [0x240000403:0x54:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee756 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 69 lru_type: 1 00000020:00200000:3.0:1775764770.764553:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1775764770.764562:0:19724:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2ba:0x0] updating i_size 260096 i_blocks 515 00000020:00010000:3.0:1775764770.764566:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef50db0 (ffff8f99703141b8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764770.764569:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef50db0 (ffff8f99703141b8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.764572:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d5cb0 (ffff8f9970314128/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.764576:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef50db0 (ffff8f99703141b8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764770.764579:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d5cb0 (ffff8f99703141b8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764770.764595:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2ba:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764770.764678:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.764682:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.764685:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:3.0:1775764770.764688:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.764690:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=16 00000080:00200000:3.0:1775764770.764693:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.764695:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:3.0:1775764770.764697:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.764700:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=16 00000080:00200000:3.0:1775764770.764703:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.764708:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=COURSES.DB, flags=0 00000080:00200000:3.0:1775764770.764711:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 3 00000080:00200000:3.0:1775764770.764716:0:19724:0:(namei.c:2395:ll_unlink()) VFS Op:name=COURSES.DB, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8) 00010000:00010000:3.0:1775764770.765017:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941481000/0x36f9bdc9e421b9a0 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ba:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f711b expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 1 00000080:00200000:3.0:1775764770.765039:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2ba:0x0]: losing layout lock 00010000:00010000:3.0:1775764770.765078:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941481000/0x36f9bdc9e421b9a0 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2ba:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f711b expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 1 00010000:00010000:3.0:1775764770.765085:0:19724:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764770.765088:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941481000/0x36f9bdc9e421b9a0 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2ba:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f711b expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 1 00000020:00100000:3.0:1775764770.765110:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1775764770.765118:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970edea00 x1862022966921984/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764770.765144:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966921984:192.168.202.146@tcp:36:dbench.0 00000100:00100000:3.0:1775764770.765205:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99707ef900 going to sleep for 11 seconds 00000100:00100000:3.0:1775764770.770018:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966921984:192.168.202.146@tcp:36:dbench.0 00000080:00200000:3.0:1775764770.770087:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2ba:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1775764770.770097:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764770.770112:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:3.0:1775764770.770113:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000008:00010000:3.0:1775764770.770127:0:19724:0:(osc_cache.c:1167:osc_extent_make_ready()) ### extent: 00000000f952f377 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000083f2c617/0x36f9bdc9e41caaf4 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x54:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee756 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 69 lru_type: 1 00000008:00010000:3.0:1775764770.770149:0:19724:0:(osc_cache.c:901:osc_extent_finish()) ### extent: 00000000f952f377 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000083f2c617/0x36f9bdc9e41caaf4 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x54:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee756 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 69 lru_type: 1 00000008:00010000:3.0:1775764770.770168:0:19724:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000f952f377 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000083f2c617/0x36f9bdc9e41caaf4 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x54:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee756 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 69 lru_type: 1 00000080:00200000:3.0:1775764770.770192:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2ba:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764770.770459:0:19724:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2ba:0x0](ffff8f9952d8d348) 00000008:00010000:3.0:1775764770.770491:0:19724:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 515 [cma]time 1775764768 1775764768 0 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000083f2c617/0x36f9bdc9e41caaf4 lrc: 1/0,0 mode: PW/PW res: [0x240000403:0x54:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee756 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 69 lru_type: 1 00000080:00200000:3.0:1775764770.770555:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.770559:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.770564:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:3.0:1775764770.770568:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.770571:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:3.0:1775764770.770573:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.770575:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:3.0:1775764770.770577:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.770579:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:3.0:1775764770.770580:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.770590:0:19724:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=ENROLL.DB, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), flags=1 00000080:00200000:3.0:1775764770.770593:0:19724:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=ENROLL.DB, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), intent=getattr 00010000:00010000:3.0:1775764770.770609:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5a00/0x36f9bdc9e421c32b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2d1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8f75 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:3.0:1775764770.770631:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c32b from it ffffafbac1c5b900 00010000:00010000:3.0:1775764770.770636:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5a00/0x36f9bdc9e421c32b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2d1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8f75 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764770.770645:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5a00/0x36f9bdc9e421c32b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8f75 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:3.0:1775764770.770672:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file ENROLL.DB:[0x200000401:0x2d1:0x0](ffff8f9952d942c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764770.770678:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2d1:0x0](ffff8f9952d942c8),name=ENROLL.DB 00000002:00010000:3.0:1775764770.770695:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2d1:0x0]) in obj [0x200000401:0x2b8:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764770.770703:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421bfe3 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2d1:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8446 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764770.770711:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421bfe3 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2d1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8446 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764770.770716:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2b8:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764770.770720:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2d1:0x0](ffff8f9952d942c8) 00000080:00010000:3.0:1775764770.770722:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2d1:0x0](ffff8f9952d942c8) for lock 0x36f9bdc9e421bfe3 00000080:00010000:3.0:1775764770.770891:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421bfe3 from it ffffafbac1c5bcb8 00010000:00010000:3.0:1775764770.770893:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421bfe3 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2d1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8446 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764770.770898:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421bfe3 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8446 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000100:00100000:3.0:1775764770.780992:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970d9df80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966922496:192.168.202.146@tcp:36:dbench.0 00000080:00200000:3.0:1775764770.781047:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.781052:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.781056:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:3.0:1775764770.781060:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.781063:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:3.0:1775764770.781065:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.781067:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:3.0:1775764770.781068:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.781070:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:3.0:1775764770.781072:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.781081:0:19724:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=STUDENTS.TV, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), flags=1 00000080:00200000:3.0:1775764770.781084:0:19724:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=STUDENTS.TV, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), intent=getattr 00010000:00010000:3.0:1775764770.781101:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4200/0x36f9bdc9e421c332 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2db:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8f91 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:3.0:1775764770.781112:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c332 from it ffffafbac1c5b900 00010000:00010000:3.0:1775764770.781115:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4200/0x36f9bdc9e421c332 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2db:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8f91 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764770.781122:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4200/0x36f9bdc9e421c332 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2db:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8f91 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:3.0:1775764770.781139:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file STUDENTS.TV:[0x200000401:0x2db:0x0](ffff8f9952dd3248), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764770.781142:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2db:0x0](ffff8f9952dd3248),name=STUDENTS.TV 00000002:00010000:3.0:1775764770.781155:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2db:0x0]) in obj [0x200000401:0x2b8:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764770.781165:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fd200/0x36f9bdc9e421bc32 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2db:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f78d8 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764770.781172:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fd200/0x36f9bdc9e421bc32 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2db:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f78d8 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764770.781177:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2b8:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764770.781180:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2db:0x0](ffff8f9952dd3248) 00000080:00010000:3.0:1775764770.781182:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2db:0x0](ffff8f9952dd3248) for lock 0x36f9bdc9e421bc32 00000080:00010000:3.0:1775764770.781185:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421bc32 from it ffffafbac1c5bcb8 00010000:00010000:3.0:1775764770.781187:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fd200/0x36f9bdc9e421bc32 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2db:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f78d8 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764770.781191:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fd200/0x36f9bdc9e421bc32 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2db:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f78d8 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764770.781226:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2db:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764770.781234:0:19724:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2db:0x0] 00000008:00010000:3.0:1775764770.781255:0:19724:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1775764770.781258:0:19724:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d5130, osc lock ffff8f9966acef70, flags 0x400001002 00010000:00010000:3.0:1775764770.781272:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000054b8cc9/0x36f9bdc9e41caf93 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x64:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef2af expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 0 00010000:00010000:3.0:1775764770.781280:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000054b8cc9/0x36f9bdc9e41caf93 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x64:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef2af expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 0 00000020:00200000:3.0:1775764770.781289:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1775764770.781292:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000054b8cc9/0x36f9bdc9e41caf93 lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x64:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ef2af expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 0 00010000:00010000:3.0:1775764770.781297:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000054b8cc9/0x36f9bdc9e41caf93 lrc: 4/0,0 mode: PW/PW res: [0x240000403:0x64:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef2af expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 0 00000020:00200000:3.0:1775764770.781302:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1775764770.781309:0:19724:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2db:0x0] updating i_size 260096 i_blocks 1 00000020:00010000:3.0:1775764770.781311:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef50db0 (ffff8f99703141b8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764770.781313:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef50db0 (ffff8f99703141b8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.781315:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d5130 (ffff8f9970314128/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.781318:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef50db0 (ffff8f99703141b8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764770.781320:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d5130 (ffff8f99703141b8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764770.781339:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2db:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764770.781353:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.781355:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.781357:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:3.0:1775764770.781359:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.781361:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=16 00000080:00200000:3.0:1775764770.781362:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.781364:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:3.0:1775764770.781365:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.781367:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=16 00000080:00200000:3.0:1775764770.781368:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.781371:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=STUDENTS.TV, flags=0 00000080:00200000:3.0:1775764770.781374:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 3 00000080:00200000:3.0:1775764770.781377:0:19724:0:(namei.c:2395:ll_unlink()) VFS Op:name=STUDENTS.TV, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8) 00010000:00010000:3.0:1775764770.781403:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4200/0x36f9bdc9e421c332 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2db:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f8f91 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764770.781417:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2db:0x0]: losing layout lock 00010000:00010000:3.0:1775764770.781430:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fd200/0x36f9bdc9e421bc32 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2db:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f78d8 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764770.781438:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2db:0x0]: losing layout lock 00010000:00010000:3.0:1775764770.781461:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4200/0x36f9bdc9e421c332 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2db:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f8f91 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764770.781466:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fd200/0x36f9bdc9e421bc32 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2db:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f78d8 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764770.781469:0:19724:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1775764770.781471:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc4200/0x36f9bdc9e421c332 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2db:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f8f91 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764770.781479:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fd200/0x36f9bdc9e421bc32 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2db:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f78d8 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000020:00100000:3.0:1775764770.781493:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1775764770.781498:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970edc700 x1862022966922880/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764770.781513:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966922880:192.168.202.146@tcp:36:dbench.0 00000100:00100000:3.0:1775764770.781575:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99707ef900 going to sleep for 11 seconds 00000100:00100000:1.0:1775764770.786857:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970d9ea00 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966922624:192.168.202.146@tcp:37:dbench.0 00010000:00010000:1.0:1775764770.787365:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c291 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2aa:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8c57 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764770.787376:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c291 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2aa:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c57 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00000080:00200000:1.0:1775764770.787449:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.787453:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.787458:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:1.0:1775764770.787464:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.787467:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:1.0:1775764770.787469:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.787471:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:1.0:1775764770.787473:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.787475:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=WORDPRO, flags=17 00000080:00200000:1.0:1775764770.787477:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2aa:0x0](ffff8f9952d12a08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.787478:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=RESULTS.XLS, flags=1 00000080:00200008:1.0:1775764770.787485:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file RESULTS.XLS:[0x200000401:0x2ae:0x0](ffff8f9952d8c2c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764770.790516:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2ae:0x0](ffff8f9952d8c2c8),name=RESULTS.XLS 00000002:00010000:1.0:1775764770.790532:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2ae:0x0]) in obj [0x200000401:0x2aa:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764770.790544:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b8600/0x36f9bdc9e4216a69 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2ae:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e6ac5 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764770.790554:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b8600/0x36f9bdc9e4216a69 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2ae:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e6ac5 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764770.790560:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2aa:0x0] suppgids 0 0: rc 1 00010000:00010000:0.0:1775764770.790561:0:19865:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971226800/0x36f9bdc9e4216c99 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e7186 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00010000:1.0:1775764770.790565:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2ae:0x0](ffff8f9952d8c2c8) 00000080:00010000:1.0:1775764770.790568:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2ae:0x0](ffff8f9952d8c2c8) for lock 0x36f9bdc9e4216a69 00010000:00010000:0.0:1775764770.790571:0:19865:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971226800/0x36f9bdc9e4216c99 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e7186 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00010000:1.0:1775764770.790573:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e4216a69 from it ffffafbac1c7bcb8 00010000:00010000:1.0:1775764770.790576:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b8600/0x36f9bdc9e4216a69 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2ae:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5e6ac5 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00010000:0.0:1775764770.790578:0:19865:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e4216c99 from it ffffafbac1effd48 00010000:00010000:0.0:1775764770.790581:0:19865:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971226800/0x36f9bdc9e4216c99 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5e7186 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764770.790584:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b8600/0x36f9bdc9e4216a69 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ae:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e6ac5 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:0.0:1775764770.790587:0:19865:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971226800/0x36f9bdc9e4216c99 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e7186 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764770.790614:0:19865:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b8e00/0x36f9bdc9e42168ef lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2ac:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e661f expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 8 lru_type: 0 00010000:00010000:0.0:1775764770.790635:0:19865:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b8e00/0x36f9bdc9e42168ef lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2ac:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e661f expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 8 lru_type: 0 00000080:00010000:0.0:1775764770.790642:0:19865:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e42168ef from it ffffafbac1effd48 00010000:00010000:0.0:1775764770.790645:0:19865:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b8e00/0x36f9bdc9e42168ef lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2ac:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5e661f expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 8 lru_type: 0 00010000:00010000:0.0:1775764770.790652:0:19865:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b8e00/0x36f9bdc9e42168ef lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ac:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e661f expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 9 lru_type: 1 00000080:00200000:3.0:1775764770.790696:0:19866:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2b0:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764770.790704:0:19866:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2b0:0x0] 00000008:00010000:3.0:1775764770.790733:0:19866:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:3.0:1775764770.790759:0:19866:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d5130, osc lock ffff8f9966acef70, flags 0x400041012 00010000:00010000:3.0:1775764770.790774:0:19866:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000004fc99e26/0x36f9bdc9e41ca988 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x50:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee358 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 89 lru_type: 1 00010000:00010000:0.0:1775764770.790801:0:19866:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000004fc99e26/0x36f9bdc9e41ca988 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x50:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee358 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 89 lru_type: 1 00010000:00010000:0.0:1775764770.790812:0:19866:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000004fc99e26/0x36f9bdc9e41ca988 lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x50:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ee358 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 89 lru_type: 1 00010000:00010000:0.0:1775764770.790819:0:19866:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000004fc99e26/0x36f9bdc9e41ca988 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x50:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee358 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 90 lru_type: 1 00000020:00010000:0.0:1775764770.790831:0:19866:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef50db0 (ffff8f99703141b8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764770.790834:0:19866:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef50db0 (ffff8f99703141b8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.790837:0:19866:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d5130 (ffff8f9970314128/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.790840:0:19866:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef50db0 (ffff8f99703141b8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764770.790843:0:19866:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d5130 (ffff8f99703141b8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764770.790866:0:19866:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2b0:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764770.790895:0:19866:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2ac:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1775764770.790900:0:19866:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2ac:0x0] 00000008:00010000:0.0:1775764770.790918:0:19866:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:0.0:1775764770.790921:0:19866:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0dfb30, osc lock ffff8f996686f430, flags 0x400041012 00010000:00010000:0.0:1775764770.790930:0:19866:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000fd4caace/0x36f9bdc9e41ca8ee lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x4e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee1d0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 50 lru_type: 1 00010000:00010000:0.0:1775764770.790940:0:19866:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000fd4caace/0x36f9bdc9e41ca8ee lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x4e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee1d0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 50 lru_type: 1 00010000:00010000:0.0:1775764770.790948:0:19866:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000fd4caace/0x36f9bdc9e41ca8ee lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x4e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ee1d0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 50 lru_type: 1 00010000:00010000:0.0:1775764770.790955:0:19866:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000fd4caace/0x36f9bdc9e41ca8ee lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x4e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee1d0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 51 lru_type: 1 00000020:00010000:0.0:1775764770.790965:0:19866:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_release():245 00000020:00010000:0.0:1775764770.790968:0:19866:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.790970:0:19866:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0dfb30 (ffff8f9966dc2128/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.790973:0:19866:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764770.790975:0:19866:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0dfb30 (ffff8f9966dc2368/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764770.790992:0:19866:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2ac:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764770.791048:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2ae:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764770.791053:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2ae:0x0] 00000008:00010000:1.0:1775764770.791071:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764770.791073:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f1c7130, osc lock ffff8f995875db50, flags 0x400001002 00010000:00010000:1.0:1775764770.791083:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000e5cf32fa/0x36f9bdc9e41ca942 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x4f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee2b0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 102 lru_type: 1 00010000:00010000:1.0:1775764770.791092:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000e5cf32fa/0x36f9bdc9e41ca942 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x4f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee2b0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 102 lru_type: 1 00000020:00200000:1.0:1775764770.791100:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764770.791103:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000e5cf32fa/0x36f9bdc9e41ca942 lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x4f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ee2b0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 102 lru_type: 1 00010000:00010000:1.0:1775764770.791110:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000e5cf32fa/0x36f9bdc9e41ca942 lrc: 4/0,0 mode: PW/PW res: [0x280000403:0x4f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee2b0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 103 lru_type: 1 00000020:00200000:1.0:1775764770.791119:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764770.791126:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2ae:0x0] updating i_size 260096 i_blocks 1 00000020:00010000:1.0:1775764770.791129:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_release():245 00000020:00010000:1.0:1775764770.791131:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764770.791133:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f1c7130 (ffff8f99705b8ea8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764770.791136:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764770.791138:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f1c7130 (ffff8f99705b8128/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764770.791155:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2ae:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764770.791180:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.791183:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.791186:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:1.0:1775764770.791189:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.791191:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=16 00000080:00200000:1.0:1775764770.791193:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.791195:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:1.0:1775764770.791197:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.791199:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=WORDPRO, flags=16 00000080:00200000:1.0:1775764770.791200:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2aa:0x0](ffff8f9952d12a08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.791203:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=RESULTS.XLS, flags=0 00000080:00200000:1.0:1775764770.791214:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2aa:0x0](ffff8f9952d12a08), inode mode 41c0 mask 3 00000080:00200000:1.0:1775764770.791219:0:19723:0:(namei.c:2395:ll_unlink()) VFS Op:name=RESULTS.XLS, dir=[0x200000401:0x2aa:0x0](ffff8f9952d12a08) 00010000:00010000:1.0:1775764770.791251:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c291 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2aa:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f8c57 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1775764770.791264:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d9000/0x36f9bdc9e41ca8cb lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2aa:0x0].0x0 bits 0x5/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ee18a expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 115 lru_type: 1 00010000:00010000:1.0:1775764770.791273:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d9e00/0x36f9bdc9e41ca911 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2aa:0x0].0x0 bits 0x11/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ee255 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 35 lru_type: 1 00010000:00010000:1.0:1775764770.791288:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b8600/0x36f9bdc9e4216a69 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ae:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5e6ac5 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764770.791300:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2ae:0x0]: losing layout lock 00010000:00010000:1.0:1775764770.791325:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b8600/0x36f9bdc9e4216a69 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2ae:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5e6ac5 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764770.791330:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c291 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2aa:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8c57 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1775764770.791334:0:19723:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1775764770.791335:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b8600/0x36f9bdc9e4216a69 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2ae:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5e6ac5 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764770.791350:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c291 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2aa:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8c57 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00000020:00100000:1.0:1775764770.791359:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 2 is allocated opc 36, max 7 00000100:00100000:1.0:1775764770.791366:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970c03b80 x1862022966923392/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764770.791379:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970c03b80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966923392:192.168.202.146@tcp:36:dbench.0 00000100:00100000:1.0:1775764770.791820:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f997130d480 going to sleep for 11 seconds 00000100:00100000:1.0:1775764770.796393:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970c03b80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966923392:192.168.202.146@tcp:36:dbench.0 00000080:00200000:1.0:1775764770.796459:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2ae:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764770.796473:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:1.0:1775764770.796495:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:1.0:1775764770.796497:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:1.0:1775764770.796533:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2ae:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764770.797841:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.797852:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.797858:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:3.0:1775764770.797865:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.797868:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:3.0:1775764770.797871:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.797874:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:3.0:1775764770.797877:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.797880:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:3.0:1775764770.797883:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.797897:0:19724:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=ENROLL.VAL, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), flags=1 00000080:00200000:3.0:1775764770.797902:0:19724:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=ENROLL.VAL, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), intent=getattr 00010000:00010000:3.0:1775764770.797923:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0dc00/0x36f9bdc9e421c340 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2d9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9008 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:3.0:1775764770.797940:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c340 from it ffffafbac1c5b900 00010000:00010000:3.0:1775764770.797955:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0dc00/0x36f9bdc9e421c340 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2d9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9008 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764770.797970:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0dc00/0x36f9bdc9e421c340 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9008 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:3.0:1775764770.798000:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file ENROLL.VAL:[0x200000401:0x2d9:0x0](ffff8f9952dd1148), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764770.798006:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2d9:0x0](ffff8f9952dd1148),name=ENROLL.VAL 00000002:00010000:3.0:1775764770.798024:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2d9:0x0]) in obj [0x200000401:0x2b8:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764770.798036:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d8000/0x36f9bdc9e421c092 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2d9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8653 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764770.798048:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d8000/0x36f9bdc9e421c092 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2d9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8653 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764770.798055:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2b8:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764770.798062:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2d9:0x0](ffff8f9952dd1148) 00000080:00010000:3.0:1775764770.798065:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2d9:0x0](ffff8f9952dd1148) for lock 0x36f9bdc9e421c092 00000080:00010000:3.0:1775764770.798070:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c092 from it ffffafbac1c5bcb8 00010000:00010000:3.0:1775764770.798074:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d8000/0x36f9bdc9e421c092 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2d9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8653 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764770.798081:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d8000/0x36f9bdc9e421c092 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8653 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764770.798123:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2d9:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764770.798132:0:19724:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2d9:0x0] 00000008:00010000:3.0:1775764770.798156:0:19724:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1775764770.798160:0:19724:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d5630, osc lock ffff8f9966ace4c0, flags 0x400001002 00010000:00010000:3.0:1775764770.798176:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000008eb5a6c2/0x36f9bdc9e41caf4d lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x63:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef207 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 57 lru_type: 1 00010000:00010000:3.0:1775764770.798187:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000008eb5a6c2/0x36f9bdc9e41caf4d lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x63:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef207 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 57 lru_type: 1 00000020:00200000:3.0:1775764770.798198:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1775764770.798201:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000008eb5a6c2/0x36f9bdc9e41caf4d lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x63:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ef207 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 57 lru_type: 1 00010000:00010000:3.0:1775764770.798208:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000008eb5a6c2/0x36f9bdc9e41caf4d lrc: 4/0,0 mode: PW/PW res: [0x240000403:0x63:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef207 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 58 lru_type: 1 00000020:00200000:3.0:1775764770.798217:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1775764770.798225:0:19724:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2d9:0x0] updating i_size 260096 i_blocks 1 00000020:00010000:3.0:1775764770.798229:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef50000 (ffff8f99703147e8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764770.798231:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef50000 (ffff8f99703147e8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.798234:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d5630 (ffff8f9970314b48/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.798237:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef50000 (ffff8f99703147e8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764770.798240:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d5630 (ffff8f99703147e8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764770.798261:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2d9:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764770.798279:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.798283:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.798286:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:3.0:1775764770.798290:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.798292:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=16 00000080:00200000:3.0:1775764770.798295:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.798298:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:3.0:1775764770.798300:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.798302:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=16 00000080:00200000:3.0:1775764770.798304:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.798309:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=ENROLL.VAL, flags=0 00000080:00200000:1.0:1775764770.798311:0:19723:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2ae:0x0](ffff8f9952d8c2c8) 00000080:00200000:3.0:1775764770.798311:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 3 00000080:00200000:3.0:1775764770.798315:0:19724:0:(namei.c:2395:ll_unlink()) VFS Op:name=ENROLL.VAL, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8) 00010000:00010000:3.0:1775764770.798350:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0dc00/0x36f9bdc9e421c340 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f9008 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000008:00010000:1.0:1775764770.798351:0:19723:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 1 [cma]time 1775764055 1775764055 0 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000e5cf32fa/0x36f9bdc9e41ca942 lrc: 1/0,0 mode: PW/PW res: [0x280000403:0x4f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee2b0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 103 lru_type: 1 00000080:00200000:3.0:1775764770.798371:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2d9:0x0]: losing layout lock 00010000:00010000:3.0:1775764770.798404:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d8000/0x36f9bdc9e421c092 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f8653 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764770.798413:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.798417:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.798421:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:3.0:1775764770.798422:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2d9:0x0]: losing layout lock 00000080:00200000:1.0:1775764770.798424:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.798427:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:1.0:1775764770.798430:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.798434:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:1.0:1775764770.798437:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.798439:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=WORDPRO, flags=17 00000080:00200000:1.0:1775764770.798441:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2aa:0x0](ffff8f9952d12a08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.798444:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=NEWS1_1.LWP, flags=1 00010000:00010000:1.0:1775764770.798452:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971226800/0x36f9bdc9e4216c99 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5e7186 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1775764770.798454:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0dc00/0x36f9bdc9e421c340 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2d9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f9008 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00010000:1.0:1775764770.798460:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e4216c99 from it ffffafbac1c7baf8 00010000:00010000:3.0:1775764770.798462:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d8000/0x36f9bdc9e421c092 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2d9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f8653 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764770.798463:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971226800/0x36f9bdc9e4216c99 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5e7186 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1775764770.798469:0:19724:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1775764770.798470:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971226800/0x36f9bdc9e4216c99 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e7186 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1775764770.798471:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0dc00/0x36f9bdc9e421c340 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2d9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f9008 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764770.798482:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d8000/0x36f9bdc9e421c092 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2d9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f8653 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000020:00100000:3.0:1775764770.798502:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1775764770.798512:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970edf100 x1862022966923648/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764770.798536:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966923648:192.168.202.146@tcp:36:dbench.0 00000100:00100000:3.0:1775764770.798639:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99707ef600 going to sleep for 11 seconds 00000080:00200008:1.0:1775764770.799275:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file NEWS1_1.LWP:[0x200000401:0x2b0:0x0](ffff8f9952d15348), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764770.799282:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2b0:0x0](ffff8f9952d15348),name=NEWS1_1.LWP 00000002:00010000:1.0:1775764770.799294:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2b0:0x0]) in obj [0x200000401:0x2aa:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764770.799304:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971226800/0x36f9bdc9e4216c99 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e7186 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764770.799312:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971226800/0x36f9bdc9e4216c99 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e7186 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764770.799319:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2aa:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764770.799323:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2b0:0x0](ffff8f9952d15348) 00000080:00010000:1.0:1775764770.799326:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2b0:0x0](ffff8f9952d15348) for lock 0x36f9bdc9e4216c99 00000080:00010000:1.0:1775764770.799330:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e4216c99 from it ffffafbac1c7bcb8 00010000:00010000:1.0:1775764770.799333:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971226800/0x36f9bdc9e4216c99 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5e7186 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764770.799340:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971226800/0x36f9bdc9e4216c99 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e7186 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00000080:00200000:1.0:1775764770.799369:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2b0:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764770.799375:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2b0:0x0] 00000008:00010000:1.0:1775764770.799395:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764770.799397:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f1c7130, osc lock ffff8f995875db50, flags 0x400001002 00010000:00010000:1.0:1775764770.799409:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000004fc99e26/0x36f9bdc9e41ca988 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x50:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee358 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 90 lru_type: 1 00010000:00010000:1.0:1775764770.799419:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000004fc99e26/0x36f9bdc9e41ca988 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x50:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee358 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 90 lru_type: 1 00000020:00200000:1.0:1775764770.799429:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764770.799433:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000004fc99e26/0x36f9bdc9e41ca988 lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x50:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ee358 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 90 lru_type: 1 00010000:00010000:1.0:1775764770.799441:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000004fc99e26/0x36f9bdc9e41ca988 lrc: 4/0,0 mode: PW/PW res: [0x280000403:0x50:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee358 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 91 lru_type: 1 00000020:00200000:1.0:1775764770.800605:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764770.800627:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2b0:0x0] updating i_size 260096 i_blocks 1 00000020:00010000:1.0:1775764770.800633:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_release():245 00000020:00010000:1.0:1775764770.800636:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764770.800640:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f1c7130 (ffff8f99705b8ea8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764770.800645:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764770.800648:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f1c7130 (ffff8f99705b8128/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764770.800679:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2b0:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764770.800712:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.800717:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.800722:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:1.0:1775764770.801471:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.801483:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=16 00000080:00200000:1.0:1775764770.801487:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.801491:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:1.0:1775764770.801493:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.801497:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=WORDPRO, flags=16 00000080:00200000:1.0:1775764770.801499:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2aa:0x0](ffff8f9952d12a08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.801506:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=NEWS1_1.LWP, flags=0 00000080:00200000:1.0:1775764770.801509:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2aa:0x0](ffff8f9952d12a08), inode mode 41c0 mask 3 00000080:00200000:1.0:1775764770.801513:0:19723:0:(namei.c:2395:ll_unlink()) VFS Op:name=NEWS1_1.LWP, dir=[0x200000401:0x2aa:0x0](ffff8f9952d12a08) 00010000:00010000:1.0:1775764770.801579:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971226800/0x36f9bdc9e4216c99 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5e7186 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00000080:00200000:1.0:1775764770.801603:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2b0:0x0]: losing layout lock 00010000:00010000:1.0:1775764770.801662:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971226800/0x36f9bdc9e4216c99 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2b0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5e7186 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764770.801669:0:19723:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764770.801672:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971226800/0x36f9bdc9e4216c99 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2b0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5e7186 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00000020:00100000:1.0:1775764770.801697:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 2 is allocated opc 36, max 7 00000100:00100000:1.0:1775764770.801722:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970b50000 x1862022966923904/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764770.801763:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970b50000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966923904:192.168.202.146@tcp:36:dbench.0 00000100:00100000:1.0:1775764770.802501:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f997130d480 going to sleep for 11 seconds 00000100:00100000:3.0:1775764770.802927:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966923648:192.168.202.146@tcp:36:dbench.0 00000080:00200000:0.0:1775764770.803086:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.803092:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.803097:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:0.0:1775764770.803102:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.803104:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:0.0:1775764770.803107:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.803110:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:0.0:1775764770.803113:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.803115:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:0.0:1775764770.803119:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.803131:0:19724:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=ENROLL.X03, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), flags=1 00000080:00200000:0.0:1775764770.803136:0:19724:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=ENROLL.X03, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), intent=getattr 00010000:00010000:0.0:1775764770.803153:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c200/0x36f9bdc9e421c347 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2d5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9024 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:0.0:1775764770.803165:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c347 from it ffffafbac1c5b900 00010000:00010000:0.0:1775764770.803169:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c200/0x36f9bdc9e421c347 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2d5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9024 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764770.803178:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c200/0x36f9bdc9e421c347 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9024 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:0.0:1775764770.803204:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file ENROLL.X03:[0x200000401:0x2d5:0x0](ffff8f9952e2ba88), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764770.803208:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2d5:0x0](ffff8f9952e2ba88),name=ENROLL.X03 00000002:00010000:0.0:1775764770.803220:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2d5:0x0]) in obj [0x200000401:0x2b8:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764770.803231:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8ca00/0x36f9bdc9e421c037 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2d5:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8550 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.803240:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8ca00/0x36f9bdc9e421c037 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2d5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8550 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764770.803247:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2b8:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764770.803253:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2d5:0x0](ffff8f9952e2ba88) 00000080:00010000:0.0:1775764770.803255:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2d5:0x0](ffff8f9952e2ba88) for lock 0x36f9bdc9e421c037 00000080:00010000:0.0:1775764770.803260:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c037 from it ffffafbac1c5bcb8 00010000:00010000:0.0:1775764770.803262:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8ca00/0x36f9bdc9e421c037 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2d5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8550 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.803274:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8ca00/0x36f9bdc9e421c037 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8550 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764770.803316:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2d5:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1775764770.803326:0:19724:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2d5:0x0] 00000008:00010000:0.0:1775764770.803352:0:19724:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1775764770.803355:0:19724:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0dfb30, osc lock ffff8f996686f430, flags 0x400001002 00010000:00010000:0.0:1775764770.803370:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c23797f9/0x36f9bdc9e41caec8 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x61:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef0b7 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 37 lru_type: 1 00010000:00010000:0.0:1775764770.803382:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c23797f9/0x36f9bdc9e41caec8 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x61:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef0b7 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 37 lru_type: 1 00000020:00200000:0.0:1775764770.803488:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1775764770.803492:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c23797f9/0x36f9bdc9e41caec8 lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x61:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ef0b7 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 37 lru_type: 1 00010000:00010000:0.0:1775764770.803501:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c23797f9/0x36f9bdc9e41caec8 lrc: 4/0,0 mode: PW/PW res: [0x240000403:0x61:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef0b7 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 38 lru_type: 1 00000020:00200000:0.0:1775764770.803512:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1775764770.803521:0:19724:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2d5:0x0] updating i_size 260096 i_blocks 1 00000020:00010000:0.0:1775764770.803524:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_release():245 00000020:00010000:0.0:1775764770.803527:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.803530:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0dfb30 (ffff8f9966dc2128/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.803534:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764770.803538:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0dfb30 (ffff8f9966dc2368/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764770.803561:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2d5:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764770.803580:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.803584:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.803587:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:0.0:1775764770.803590:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.803592:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=16 00000080:00200000:0.0:1775764770.803595:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.803597:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:0.0:1775764770.803599:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.804011:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=16 00000080:00200000:0.0:1775764770.804014:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.804018:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=ENROLL.X03, flags=0 00000080:00200000:0.0:1775764770.804021:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 3 00000080:00200000:0.0:1775764770.804024:0:19724:0:(namei.c:2395:ll_unlink()) VFS Op:name=ENROLL.X03, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8) 00010000:00010000:0.0:1775764770.804054:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c200/0x36f9bdc9e421c347 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d5:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f9024 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1775764770.804068:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2d5:0x0]: losing layout lock 00010000:00010000:0.0:1775764770.804082:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8ca00/0x36f9bdc9e421c037 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d5:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f8550 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764770.804097:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2d5:0x0]: losing layout lock 00010000:00010000:0.0:1775764770.804126:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c200/0x36f9bdc9e421c347 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2d5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f9024 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764770.804133:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8ca00/0x36f9bdc9e421c037 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2d5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f8550 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764770.804144:0:19724:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1775764770.804147:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0c200/0x36f9bdc9e421c347 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2d5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f9024 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764770.804158:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8ca00/0x36f9bdc9e421c037 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2d5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f8550 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000020:00100000:0.0:1775764770.804174:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1775764770.804180:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f0a8380 x1862022966924160/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764770.804195:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0a8380 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966924160:192.168.202.146@tcp:36:dbench.0 00000100:00100000:0.0:1775764770.804242:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f600 going to sleep for 11 seconds 00000100:00100000:1.0:1775764770.814239:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0a8380 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966924160:192.168.202.146@tcp:36:dbench.0 00000080:00200000:1.0:1775764770.814330:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.814337:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.814344:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:1.0:1775764770.814351:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.814355:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:1.0:1775764770.814358:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.814361:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:1.0:1775764770.814363:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.814366:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:1.0:1775764770.814368:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.814383:0:19724:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=STUDENTS.MB, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), flags=1 00000080:00200000:1.0:1775764770.814395:0:19724:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=STUDENTS.MB, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), intent=getattr 00010000:00010000:1.0:1775764770.814415:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0d800/0x36f9bdc9e421c34e lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2cf:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9040 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:1.0:1775764770.814432:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c34e from it ffffafbac1c5b900 00010000:00010000:1.0:1775764770.814436:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0d800/0x36f9bdc9e421c34e lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2cf:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9040 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764770.814445:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0d800/0x36f9bdc9e421c34e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2cf:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9040 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:1.0:1775764770.814477:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file STUDENTS.MB:[0x200000401:0x2cf:0x0](ffff8f9952e2d348), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764770.814483:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2cf:0x0](ffff8f9952e2d348),name=STUDENTS.MB 00000002:00010000:1.0:1775764770.814500:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2cf:0x0]) in obj [0x200000401:0x2b8:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764770.814513:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d9600/0x36f9bdc9e421c0c3 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2cf:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f86ed expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.814523:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d9600/0x36f9bdc9e421c0c3 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2cf:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f86ed expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764770.814531:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2b8:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764770.814536:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2cf:0x0](ffff8f9952e2d348) 00000080:00010000:1.0:1775764770.814539:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2cf:0x0](ffff8f9952e2d348) for lock 0x36f9bdc9e421c0c3 00000080:00010000:1.0:1775764770.814544:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c0c3 from it ffffafbac1c5bcb8 00010000:00010000:1.0:1775764770.814547:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d9600/0x36f9bdc9e421c0c3 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2cf:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f86ed expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764770.814554:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d9600/0x36f9bdc9e421c0c3 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2cf:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f86ed expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764770.814602:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2cf:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000100:00100000:3.0:1775764770.814602:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970b50000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966923904:192.168.202.146@tcp:36:dbench.0 00000001:00010000:1.0:1775764770.814614:0:19724:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2cf:0x0] 00000080:00200000:3.0:1775764770.814664:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2b0:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1775764770.814673:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764770.814693:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:3.0:1775764770.814695:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:3.0:1775764770.815312:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2b0:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:1.0:1775764770.815638:0:19724:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764770.815643:0:19724:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f1c7130, osc lock ffff8f995875db50, flags 0x400001002 00010000:00010000:1.0:1775764770.815663:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000b672b57f/0x36f9bdc9e41cadef lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x5e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eeebf expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 57 lru_type: 1 00010000:00010000:1.0:1775764770.815676:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000b672b57f/0x36f9bdc9e41cadef lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x5e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eeebf expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 57 lru_type: 1 00000020:00200000:1.0:1775764770.815692:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764770.815695:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000b672b57f/0x36f9bdc9e41cadef lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x5e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4eeebf expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 57 lru_type: 1 00010000:00010000:1.0:1775764770.815703:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000b672b57f/0x36f9bdc9e41cadef lrc: 4/0,0 mode: PW/PW res: [0x240000403:0x5e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eeebf expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 58 lru_type: 1 00000020:00200000:1.0:1775764770.815715:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764770.815724:0:19724:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2cf:0x0] updating i_size 260096 i_blocks 1 00000020:00010000:1.0:1775764770.815728:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_release():245 00000020:00010000:1.0:1775764770.815731:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764770.815733:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f1c7130 (ffff8f99705b8ea8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764770.815751:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764770.815754:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f1c7130 (ffff8f99705b8128/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764770.815774:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2cf:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764770.815801:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.815806:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.815811:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:1.0:1775764770.815815:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.815818:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=16 00000080:00200000:1.0:1775764770.815821:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.815824:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:1.0:1775764770.815826:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.815829:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=16 00000080:00200000:1.0:1775764770.815831:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.815837:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=STUDENTS.MB, flags=0 00000080:00200000:1.0:1775764770.815841:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 3 00000080:00200000:1.0:1775764770.815846:0:19724:0:(namei.c:2395:ll_unlink()) VFS Op:name=STUDENTS.MB, dir=[0x200000401:0x2b8:0x0](ffff8f9952d880c8) 00010000:00010000:1.0:1775764770.815889:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0d800/0x36f9bdc9e421c34e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2cf:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f9040 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764770.815910:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2cf:0x0]: losing layout lock 00010000:00010000:1.0:1775764770.815929:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d9600/0x36f9bdc9e421c0c3 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2cf:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f86ed expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764770.815943:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2cf:0x0]: losing layout lock 00010000:00010000:1.0:1775764770.815980:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0d800/0x36f9bdc9e421c34e lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2cf:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f9040 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764770.815987:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d9600/0x36f9bdc9e421c0c3 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2cf:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f86ed expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764770.815993:0:19724:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1775764770.815996:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0d800/0x36f9bdc9e421c34e lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2cf:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f9040 expref: -99 pid: 19861 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764770.816008:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d9600/0x36f9bdc9e421c0c3 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2cf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f86ed expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000020:00100000:1.0:1775764770.816028:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1775764770.816037:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970b52300 x1862022966924416/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764770.816058:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970b52300 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966924416:192.168.202.146@tcp:36:dbench.0 00000100:00100000:1.0:1775764770.816222:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f997130d900 going to sleep for 11 seconds 00000080:00200000:0.0:1775764770.819402:0:19723:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2b0:0x0](ffff8f9952d15348) 00000008:00010000:0.0:1775764770.819475:0:19723:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 1 [cma]time 1775764055 1775764055 0 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000004fc99e26/0x36f9bdc9e41ca988 lrc: 1/0,0 mode: PW/PW res: [0x280000403:0x50:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee358 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 91 lru_type: 1 00000080:00200000:0.0:1775764770.819578:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.819586:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.819593:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:0.0:1775764770.819601:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.822567:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:0.0:1775764770.822571:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.822574:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:0.0:1775764770.822577:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.822580:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=WORDPRO, flags=17 00000080:00200000:0.0:1775764770.822583:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2aa:0x0](ffff8f9952d12a08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.822587:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=BENCHS.LWP, flags=1 00010000:00010000:0.0:1775764770.822597:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b8e00/0x36f9bdc9e42168ef lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2ac:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5e661f expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 9 lru_type: 1 00000080:00010000:0.0:1775764770.822676:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e42168ef from it ffffafbac1c7baf8 00010000:00010000:0.0:1775764770.822680:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b8e00/0x36f9bdc9e42168ef lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2ac:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5e661f expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 9 lru_type: 1 00010000:00010000:0.0:1775764770.822687:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b8e00/0x36f9bdc9e42168ef lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ac:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e661f expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 10 lru_type: 1 00000080:00200008:0.0:1775764770.822719:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file BENCHS.LWP:[0x200000401:0x2ac:0x0](ffff8f9952da5348), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764770.822724:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2ac:0x0](ffff8f9952da5348),name=BENCHS.LWP 00000002:00010000:0.0:1775764770.822733:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2ac:0x0]) in obj [0x200000401:0x2aa:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764770.822758:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b8e00/0x36f9bdc9e42168ef lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2ac:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e661f expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 10 lru_type: 1 00010000:00010000:0.0:1775764770.822767:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b8e00/0x36f9bdc9e42168ef lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2ac:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e661f expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 10 lru_type: 1 00000080:00200000:0.0:1775764770.822774:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2aa:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764770.822778:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2ac:0x0](ffff8f9952da5348) 00000080:00010000:0.0:1775764770.822780:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2ac:0x0](ffff8f9952da5348) for lock 0x36f9bdc9e42168ef 00000080:00010000:0.0:1775764770.822784:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e42168ef from it ffffafbac1c7bcb8 00010000:00010000:0.0:1775764770.822786:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b8e00/0x36f9bdc9e42168ef lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2ac:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5e661f expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 10 lru_type: 1 00010000:00010000:0.0:1775764770.822799:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b8e00/0x36f9bdc9e42168ef lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ac:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e661f expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 11 lru_type: 1 00000080:00200000:0.0:1775764770.822828:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2ac:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1775764770.822833:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2ac:0x0] 00000008:00010000:0.0:1775764770.822854:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1775764770.822856:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0df9b0, osc lock ffff8f996686f430, flags 0x400001002 00010000:00010000:0.0:1775764770.822867:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000fd4caace/0x36f9bdc9e41ca8ee lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x4e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee1d0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 51 lru_type: 1 00010000:00010000:0.0:1775764770.822878:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000fd4caace/0x36f9bdc9e41ca8ee lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x4e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee1d0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 51 lru_type: 1 00000020:00200000:0.0:1775764770.822886:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1775764770.822889:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000fd4caace/0x36f9bdc9e41ca8ee lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x4e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ee1d0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 51 lru_type: 1 00010000:00010000:0.0:1775764770.822897:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000fd4caace/0x36f9bdc9e41ca8ee lrc: 4/0,0 mode: PW/PW res: [0x280000403:0x4e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee1d0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 52 lru_type: 1 00000020:00200000:0.0:1775764770.822907:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1775764770.822912:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2ac:0x0] updating i_size 260096 i_blocks 1 00000020:00010000:0.0:1775764770.822915:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_release():245 00000020:00010000:0.0:1775764770.822917:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.822920:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0df9b0 (ffff8f9966dc2128/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.822922:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764770.825346:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0df9b0 (ffff8f9966dc2368/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764770.825387:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2ac:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764770.825429:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.825434:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.825439:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:0.0:1775764770.825445:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.825448:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=16 00000080:00200000:0.0:1775764770.825451:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.825454:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:0.0:1775764770.825456:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.825459:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=WORDPRO, flags=16 00000080:00200000:0.0:1775764770.825461:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2aa:0x0](ffff8f9952d12a08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.825467:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=BENCHS.LWP, flags=0 00000080:00200000:0.0:1775764770.825471:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2aa:0x0](ffff8f9952d12a08), inode mode 41c0 mask 3 00000080:00200000:0.0:1775764770.825476:0:19723:0:(namei.c:2395:ll_unlink()) VFS Op:name=BENCHS.LWP, dir=[0x200000401:0x2aa:0x0](ffff8f9952d12a08) 00010000:00010000:0.0:1775764770.828035:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b8e00/0x36f9bdc9e42168ef lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ac:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5e661f expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 11 lru_type: 1 00000080:00200000:0.0:1775764770.828072:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2ac:0x0]: losing layout lock 00010000:00010000:0.0:1775764770.828111:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b8e00/0x36f9bdc9e42168ef lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2ac:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5e661f expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 11 lru_type: 1 00010000:00010000:0.0:1775764770.828115:0:19723:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764770.828118:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b8e00/0x36f9bdc9e42168ef lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2ac:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5e661f expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 11 lru_type: 1 00000020:00100000:0.0:1775764770.828135:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1775764770.828140:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f0aaa00 x1862022966924928/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764770.828160:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0aaa00 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966924928:192.168.202.146@tcp:36:dbench.0 00000100:00100000:0.0:1775764770.828244:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f600 going to sleep for 11 seconds 00000100:00100000:0.0:1775764770.836153:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0aaa00 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966924928:192.168.202.146@tcp:36:dbench.0 00000080:00200000:0.0:1775764770.836236:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2ac:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764770.836249:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1775764770.836270:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:0.0:1775764770.836272:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:0.0:1775764770.836307:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2ac:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764770.839528:0:19723:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2ac:0x0](ffff8f9952da5348) 00000008:00010000:1.0:1775764770.839567:0:19723:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 1 [cma]time 1775764055 1775764055 0 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000fd4caace/0x36f9bdc9e41ca8ee lrc: 1/0,0 mode: PW/PW res: [0x280000403:0x4e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee1d0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 52 lru_type: 1 00000080:00200000:1.0:1775764770.842348:0:19723:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2aa:0x0](ffff8f9952d12a08) pos/size18446744073709551614/7168 32bit_api 0 00000080:00200008:1.0:1775764770.842373:0:19723:0:(file.c:447:ll_file_release()) START file WORDPRO:[0x200000401:0x2aa:0x0](ffff8f9952d12a08), flags 304000 00010000:00010000:1.0:1775764770.842880:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d9000/0x36f9bdc9e41ca8cb lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2aa:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ee18a expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 116 lru_type: 1 00000080:00200000:1.0:1775764770.842930:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.842936:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.842941:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:1.0:1775764770.842947:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.842951:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:1.0:1775764770.842954:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.842958:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:1.0:1775764770.842960:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.842964:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=EXCEL, flags=1 00010000:00010000:1.0:1775764770.842977:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c298 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8c73 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00010000:1.0:1775764770.842987:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c298 from it ffffafbac1c7baf8 00010000:00010000:1.0:1775764770.842991:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c298 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8c73 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764770.842997:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c298 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c73 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200008:1.0:1775764770.843030:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file EXCEL:[0x200000401:0x27f:0x0](ffff8f9952db5348), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764770.843036:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x27f:0x0](ffff8f9952db5348),name=EXCEL 00000002:00010000:1.0:1775764770.843054:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x27f:0x0]) in obj [0x200000401:0x26c:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764770.843062:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c298 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c73 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764770.843070:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c298 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c73 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764770.843076:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26c:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764770.843082:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x27f:0x0](ffff8f9952db5348) 00000080:00010000:1.0:1775764770.843084:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x27f:0x0](ffff8f9952db5348) for lock 0x36f9bdc9e421c298 00000080:00010000:1.0:1775764770.843088:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c298 from it ffffafbac1c7bcb8 00010000:00010000:1.0:1775764770.843090:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c298 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8c73 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764770.843094:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c298 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c73 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764770.843116:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.843118:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.843120:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=19 00000080:00200000:1.0:1775764770.843122:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764770.843123:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=19 00000080:00200000:1.0:1775764770.843124:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.843126:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=19 00000080:00200000:1.0:1775764770.843128:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764770.843130:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=EXCEL, flags=259 00000080:00200000:1.0:1775764770.843133:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x27f:0x0](ffff8f9952db5348), inode mode 41c0 mask 44 00000080:00200008:1.0:1775764770.843138:0:19723:0:(file.c:1057:ll_file_open()) START file EXCEL:[0x200000401:0x27f:0x0](ffff8f9952db5348), flags 304000 00000080:00200008:1.0:1775764770.843148:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file EXCEL:[0x200000401:0x27f:0x0](ffff8f9952db5348), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764770.843150:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x27f:0x0](ffff8f9952db5348),name=EXCEL 00000002:00010000:1.0:1775764770.843154:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x27f:0x0]) in obj [0x200000401:0x26c:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764770.843158:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c298 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c73 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764770.843163:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c298 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c73 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764770.843167:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26c:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764770.843169:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x27f:0x0](ffff8f9952db5348) 00000080:00010000:1.0:1775764770.843170:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x27f:0x0](ffff8f9952db5348) for lock 0x36f9bdc9e421c298 00000080:00010000:1.0:1775764770.843172:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c298 from it ffffafbac1c7bcd8 00010000:00010000:1.0:1775764770.843174:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c298 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8c73 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764770.843178:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c298 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c73 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764770.843187:0:19723:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x27f:0x0](ffff8f9952db5348) pos/size0/7168 32bit_api 0 00000002:00010000:1.0:1775764770.843194:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x27f:0x0]) in obj [0x200000401:0x27f:0x0], intent: readdir flags 0 00010000:00010000:1.0:1775764770.843197:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c298 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c73 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764770.843205:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c298 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c73 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000100:00100000:1.0:1775764770.843703:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970b53480 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966925568:192.168.202.146@tcp:37:dbench.0 00000100:00100000:1.0:1775764770.843836:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f997130d000 going to sleep for 11 seconds 00000100:00100000:3.0:1775764770.849581:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970b53480 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966925568:192.168.202.146@tcp:37:dbench.0 00010000:00010000:3.0:1775764770.850120:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c298 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8c73 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1775764770.850130:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c298 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8c73 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:3.0:1775764770.850170:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.850173:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.850175:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:3.0:1775764770.850178:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.850180:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:3.0:1775764770.850182:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.850184:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:3.0:1775764770.850186:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.850188:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=EXCEL, flags=17 00000080:00200000:3.0:1775764770.850190:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x27f:0x0](ffff8f9952db5348), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.850200:0:19723:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=SALES.XLS, dir=[0x200000401:0x27f:0x0](ffff8f9952db5348), flags=1 00000080:00200000:3.0:1775764770.850203:0:19723:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=SALES.XLS, dir=[0x200000401:0x27f:0x0](ffff8f9952db5348), intent=getattr 00000002:00010000:3.0:1775764770.850221:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: SALES.XLS,[0x0:0x0:0x0]) in obj [0x200000401:0x27f:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764770.850256:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3be lrc: 3/1,0 mode: --/CR res: [0x200000401:0x27f:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.850261:0:19723:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3be lrc: 3/1,0 mode: --/CR res: [0x200000401:0x27f:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.850268:0:19723:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3be lrc: 3/1,0 mode: --/CR res: [0x200000401:0x27f:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764770.850282:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966925696:192.168.202.146@tcp:101:dbench.0 00000100:00100000:3.0:1775764770.850362:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99707ef600 going to sleep for 11 seconds 00000100:00100000:3.0:1775764770.854268:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966925696:192.168.202.146@tcp:101:dbench.0 00010000:00010000:3.0:1775764770.854281:0:19723:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3be lrc: 4/1,0 mode: --/CR res: [0x200000401:0x27f:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9541 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.854290:0:19723:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3be lrc: 4/1,0 mode: --/PR res: [0x200000401:0x283:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9541 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.854295:0:19723:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3be lrc: 4/1,0 mode: --/PR res: [0x200000401:0x283:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9541 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.854303:0:19723:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3be lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x283:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9541 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.854309:0:19723:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3be lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x283:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9541 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.854317:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3be lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x283:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9541 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.854322:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3be lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x283:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9541 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.854326:0:19723:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3be lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x283:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9541 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764770.854332:0:19723:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970edce00 x1862022966925696/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/928 e 0 to 0 dl 1775764786 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'dbench.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1775764770.854338:0:19723:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3be lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x283:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9541 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:3.0:1775764770.854345:0:19723:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3be lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x283:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9541 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.854350:0:19723:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/643 (0 0) 00000080:00200000:3.0:1775764770.854353:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x27f:0x0] suppgids 0 -1: rc 0 00000080:00200000:3.0:1775764770.854471:0:19723:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x283:0x0](ffff8f9952db80c8): rc = 0 00000080:00200000:3.0:1775764770.854483:0:19723:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x283:0x0]: layout version change: 3 -> 3 00000080:00010000:3.0:1775764770.854490:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x283:0x0](ffff8f9952db80c8) for lock 0x36f9bdc9e421c3be 00000080:00010000:3.0:1775764770.854497:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x283:0x0](ffff8f9952db80c8) 00000080:00010000:3.0:1775764770.854499:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c3be from it ffffafbac1c7bb20 00010000:00010000:3.0:1775764770.854501:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3be lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x283:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9541 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.854510:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3be lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x283:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9541 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200008:3.0:1775764770.854536:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file SALES.XLS:[0x200000401:0x283:0x0](ffff8f9952db80c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764770.857153:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x283:0x0](ffff8f9952db80c8),name=SALES.XLS 00000002:00010000:3.0:1775764770.857180:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x283:0x0]) in obj [0x200000401:0x27f:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764770.857196:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3be lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x283:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9541 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764770.857209:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3be lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x283:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9541 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764770.857215:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x27f:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764770.857220:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x283:0x0](ffff8f9952db80c8) 00000080:00010000:3.0:1775764770.857223:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x283:0x0](ffff8f9952db80c8) for lock 0x36f9bdc9e421c3be 00000080:00010000:3.0:1775764770.857228:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c3be from it ffffafbac1c7bcb8 00010000:00010000:3.0:1775764770.857231:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3be lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x283:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9541 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764770.857236:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3be lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x283:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9541 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764770.857274:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x283:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764770.857282:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x283:0x0] 00000008:00010000:3.0:1775764770.857302:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1775764770.857304:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d5630, osc lock ffff8f9966ace4c0, flags 0x400001002 00010000:00010000:3.0:1775764770.857319:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000085d887c7/0x36f9bdc9e41ca314 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x3d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ed28e expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 128 lru_type: 1 00010000:00010000:3.0:1775764770.857327:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000085d887c7/0x36f9bdc9e41ca314 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x3d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ed28e expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 128 lru_type: 1 00000020:00200000:3.0:1775764770.857336:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1775764770.857339:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000085d887c7/0x36f9bdc9e41ca314 lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x3d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ed28e expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 128 lru_type: 1 00010000:00010000:3.0:1775764770.857343:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000085d887c7/0x36f9bdc9e41ca314 lrc: 4/0,0 mode: PW/PW res: [0x240000403:0x3d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ed28e expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 129 lru_type: 1 00000020:00200000:3.0:1775764770.857348:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1775764770.857357:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x283:0x0] updating i_size 260096 i_blocks 515 00000020:00010000:3.0:1775764770.857360:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef50000 (ffff8f99703147e8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764770.857363:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef50000 (ffff8f99703147e8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.857364:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d5630 (ffff8f9970314b48/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.857367:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef50000 (ffff8f99703147e8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764770.857369:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d5630 (ffff8f99703147e8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764770.857452:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x283:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764770.857482:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.857485:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.857488:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:3.0:1775764770.857493:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.857495:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=16 00000080:00200000:3.0:1775764770.857496:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.857499:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:3.0:1775764770.857501:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.857503:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=EXCEL, flags=16 00000080:00200000:3.0:1775764770.857504:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x27f:0x0](ffff8f9952db5348), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.857508:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=SALES.XLS, flags=0 00000080:00200000:3.0:1775764770.857513:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x27f:0x0](ffff8f9952db5348), inode mode 41c0 mask 3 00000080:00200000:3.0:1775764770.857517:0:19723:0:(namei.c:2395:ll_unlink()) VFS Op:name=SALES.XLS, dir=[0x200000401:0x27f:0x0](ffff8f9952db5348) 00010000:00010000:3.0:1775764770.857543:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c298 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f8c73 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:3.0:1775764770.857554:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971071e00/0x36f9bdc9e41ca29d lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x27f:0x0].0x0 bits 0x5/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ed137 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 230 lru_type: 1 00010000:00010000:3.0:1775764770.857561:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9940030000/0x36f9bdc9e41ca2ce lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x11/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ed1f4 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 83 lru_type: 1 00010000:00010000:3.0:1775764770.857601:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3be lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x283:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f9541 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764770.857703:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x283:0x0]: losing layout lock 00010000:00010000:3.0:1775764770.857716:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966bb0600/0x36f9bdc9e42165a0 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x283:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5e5bbb expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1775764770.858893:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x283:0x0]: losing layout lock 00010000:00010000:3.0:1775764770.858936:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3be lrc: 1/0,0 mode: --/PR res: [0x200000401:0x283:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f9541 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764770.858988:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966bb0600/0x36f9bdc9e42165a0 lrc: 1/0,0 mode: --/CR res: [0x200000401:0x283:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5e5bbb expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1775764770.858996:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c298 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8c73 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:3.0:1775764770.858999:0:19723:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:3.0:1775764770.859001:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3be lrc: 0/0,0 mode: --/PR res: [0x200000401:0x283:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f9541 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764770.859009:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966bb0600/0x36f9bdc9e42165a0 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x283:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5e5bbb expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1775764770.859020:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c298 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8c73 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000020:00100000:3.0:1775764770.859031:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 2 is allocated opc 36, max 7 00000100:00100000:3.0:1775764770.859041:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970edce00 x1862022966926208/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 504/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764770.859059:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966926208:192.168.202.146@tcp:36:dbench.0 00000100:00100000:3.0:1775764770.859151:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99707ef900 going to sleep for 11 seconds 00000002:00010000:1.0:1775764770.859234:0:19867:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x27f:0x0]) in obj [0x200000401:0x27f:0x0], intent: readdir flags 0 00010000:00010000:1.0:1775764770.859243:0:19867:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/639 (0 0) 00010000:00010000:1.0:1775764770.859267:0:19867:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641c00/0x36f9bdc9e421c3c5 lrc: 3/1,0 mode: --/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 19867 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.859274:0:19867:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641c00/0x36f9bdc9e421c3c5 lrc: 3/1,0 mode: --/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19867 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.859283:0:19867:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641c00/0x36f9bdc9e421c3c5 lrc: 3/1,0 mode: --/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19867 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764770.859293:0:19867:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970b52680 pname:cluuid:pid:xid:nid:opc:job ll_sa_19723:3f83546c-01a9-4156-8a97-44c446245d3e:19867:1862022966926336:192.168.202.146@tcp:101:ll_sa.0 00000100:00100000:1.0:1775764770.859322:0:19867:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f997130de40 going to sleep for 11 seconds 00000100:00100000:3.0:1775764770.866884:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966926208:192.168.202.146@tcp:36:dbench.0 00000080:00200000:3.0:1775764770.866954:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x283:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1775764770.866966:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764770.866984:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:3.0:1775764770.866986:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:3.0:1775764770.867016:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x283:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764770.868445:0:19723:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x283:0x0](ffff8f9952db80c8) 00000008:00010000:3.0:1775764770.868489:0:19723:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 515 [cma]time 1775764722 1775764709 1775764701 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000085d887c7/0x36f9bdc9e41ca314 lrc: 1/0,0 mode: PW/PW res: [0x240000403:0x3d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ed28e expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 129 lru_type: 1 00000080:00200000:3.0:1775764770.868573:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.868580:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.868585:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:3.0:1775764770.868591:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.868594:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:3.0:1775764770.868596:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.868599:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:3.0:1775764770.868601:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.868603:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=EXCEL, flags=17 00000080:00200000:3.0:1775764770.868605:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x27f:0x0](ffff8f9952db5348), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.868616:0:19723:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=PCMAGCD.XLS, dir=[0x200000401:0x27f:0x0](ffff8f9952db5348), flags=1 00000080:00200000:3.0:1775764770.868633:0:19723:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=PCMAGCD.XLS, dir=[0x200000401:0x27f:0x0](ffff8f9952db5348), intent=getattr 00000002:00010000:3.0:1775764770.868663:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: PCMAGCD.XLS,[0x0:0x0:0x0]) in obj [0x200000401:0x27f:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764770.868696:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c3cc lrc: 3/1,0 mode: --/CR res: [0x200000401:0x27f:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.868704:0:19723:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c3cc lrc: 3/1,0 mode: --/CR res: [0x200000401:0x27f:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.868712:0:19723:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c3cc lrc: 3/1,0 mode: --/CR res: [0x200000401:0x27f:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764770.868723:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966926720:192.168.202.146@tcp:101:dbench.0 00000100:00100000:3.0:1775764770.868783:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99707ef900 going to sleep for 11 seconds 00000100:00100000:3.0:1775764770.877073:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966926720:192.168.202.146@tcp:101:dbench.0 00010000:00010000:3.0:1775764770.877089:0:19723:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c3cc lrc: 4/1,0 mode: --/CR res: [0x200000401:0x27f:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9595 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.877102:0:19723:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c3cc lrc: 4/1,0 mode: --/PR res: [0x200000401:0x280:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9595 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.877107:0:19723:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c3cc lrc: 4/1,0 mode: --/PR res: [0x200000401:0x280:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9595 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.877115:0:19723:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c3cc lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x280:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9595 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.877123:0:19723:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c3cc lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x280:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9595 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.877131:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c3cc lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x280:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9595 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.877136:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c3cc lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x280:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9595 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.877141:0:19723:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c3cc lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x280:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9595 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764770.877147:0:19723:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970edc380 x1862022966926720/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/928 e 0 to 0 dl 1775764786 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'dbench.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1775764770.877155:0:19723:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c3cc lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x280:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9595 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:3.0:1775764770.877166:0:19723:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c3cc lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x280:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9595 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.877173:0:19723:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/640 (0 0) 00000080:00200000:3.0:1775764770.877177:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x27f:0x0] suppgids 0 -1: rc 0 00000080:00200000:3.0:1775764770.877246:0:19723:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x280:0x0](ffff8f9952db42c8): rc = 0 00000080:00200000:3.0:1775764770.877258:0:19723:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x280:0x0]: layout version change: 3 -> 3 00000080:00010000:3.0:1775764770.877264:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x280:0x0](ffff8f9952db42c8) for lock 0x36f9bdc9e421c3cc 00000080:00010000:3.0:1775764770.877272:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x280:0x0](ffff8f9952db42c8) 00000080:00010000:3.0:1775764770.877274:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c3cc from it ffffafbac1c7bb20 00010000:00010000:3.0:1775764770.877276:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c3cc lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x280:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9595 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.877283:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c3cc lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x280:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9595 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200008:3.0:1775764770.877315:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file PCMAGCD.XLS:[0x200000401:0x280:0x0](ffff8f9952db42c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764770.877320:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x280:0x0](ffff8f9952db42c8),name=PCMAGCD.XLS 00000002:00010000:3.0:1775764770.877337:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x280:0x0]) in obj [0x200000401:0x27f:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764770.877344:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c3cc lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x280:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9595 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764770.877352:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c3cc lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x280:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9595 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764770.877357:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x27f:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764770.877360:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x280:0x0](ffff8f9952db42c8) 00000080:00010000:3.0:1775764770.877362:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x280:0x0](ffff8f9952db42c8) for lock 0x36f9bdc9e421c3cc 00000080:00010000:3.0:1775764770.877364:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c3cc from it ffffafbac1c7bcb8 00010000:00010000:3.0:1775764770.877366:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c3cc lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x280:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9595 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764770.877370:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c3cc lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x280:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9595 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764770.877392:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x280:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764770.877399:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x280:0x0] 00000008:00010000:3.0:1775764770.877419:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1775764770.877421:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d5630, osc lock ffff8f9966ace4c0, flags 0x400001002 00010000:00010000:3.0:1775764770.877434:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000e209157c/0x36f9bdc9e41ca2b9 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x3c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ed199 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 128 lru_type: 1 00010000:00010000:3.0:1775764770.877442:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000e209157c/0x36f9bdc9e41ca2b9 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x3c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ed199 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 128 lru_type: 1 00000020:00200000:3.0:1775764770.877449:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1775764770.877452:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000e209157c/0x36f9bdc9e41ca2b9 lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x3c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ed199 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 128 lru_type: 1 00010000:00010000:3.0:1775764770.877456:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000e209157c/0x36f9bdc9e41ca2b9 lrc: 4/0,0 mode: PW/PW res: [0x240000403:0x3c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ed199 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 129 lru_type: 1 00000020:00200000:3.0:1775764770.877462:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1775764770.877469:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x280:0x0] updating i_size 260096 i_blocks 515 00000020:00010000:3.0:1775764770.877472:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef50000 (ffff8f99703147e8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764770.877474:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef50000 (ffff8f99703147e8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.877477:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d5630 (ffff8f9970314b48/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.877479:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef50000 (ffff8f99703147e8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764770.877481:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d5630 (ffff8f99703147e8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764770.877492:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x280:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764770.877517:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.877519:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.877523:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:3.0:1775764770.877526:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.877528:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=16 00000080:00200000:3.0:1775764770.877530:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.877532:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:3.0:1775764770.877534:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.877536:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=EXCEL, flags=16 00000080:00200000:3.0:1775764770.877537:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x27f:0x0](ffff8f9952db5348), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.877540:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PCMAGCD.XLS, flags=0 00000080:00200000:3.0:1775764770.877543:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x27f:0x0](ffff8f9952db5348), inode mode 41c0 mask 3 00000080:00200000:3.0:1775764770.877546:0:19723:0:(namei.c:2395:ll_unlink()) VFS Op:name=PCMAGCD.XLS, dir=[0x200000401:0x27f:0x0](ffff8f9952db5348) 00010000:00010000:3.0:1775764770.877569:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c3cc lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x280:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f9595 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764770.877582:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x280:0x0]: losing layout lock 00010000:00010000:3.0:1775764770.877594:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fd600/0x36f9bdc9e4215dc0 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x280:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5e41b3 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1775764770.877602:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x280:0x0]: losing layout lock 00010000:00010000:3.0:1775764770.877637:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c3cc lrc: 1/0,0 mode: --/PR res: [0x200000401:0x280:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f9595 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764770.877649:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fd600/0x36f9bdc9e4215dc0 lrc: 1/0,0 mode: --/CR res: [0x200000401:0x280:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5e41b3 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1775764770.877655:0:19723:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1775764770.877658:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c3cc lrc: 0/0,0 mode: --/PR res: [0x200000401:0x280:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f9595 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764770.877670:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fd600/0x36f9bdc9e4215dc0 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x280:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5e41b3 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000020:00100000:3.0:1775764770.877688:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1775764770.877693:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970edc380 x1862022966926976/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764770.877711:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966926976:192.168.202.146@tcp:36:dbench.0 00000100:00100000:3.0:1775764770.877788:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99707ef900 going to sleep for 11 seconds 00000100:00100000:0.0:1775764770.879366:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764770.879442:0:5561:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1775764770.879455:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978358272 00000100:00100000:1.0:1775764770.879461:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978358272 00000100:00100000:1.0:1775764770.879501:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24104 00000100:00100000:1.0:1775764770.879508:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9966933850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+5:9373:x1862022978358272:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764770.879518:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641c00/0x36f9bdc9e421c3c5 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x2aac6a50e5f9579 expref: -99 pid: 19867 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.879570:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764770.879574:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9966933850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+5:9373:x1862022978358272:12345-192.168.202.146@tcp:104: Request processed in 67us (214us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764770.879578:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24104 00010000:00010000:3.0:1775764770.879830:0:19800:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1775764770.879836:0:19800:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641c00/0x36f9bdc9e421c3c5 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x2aac6a50e5f9579 expref: -99 pid: 19867 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.879842:0:19800:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) Lock ffff8f9970641c00 is referenced, will be cancelled later 00010000:00010000:3.0:1775764770.879844:0:19800:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641c00/0x36f9bdc9e421c3c5 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x2/0x2 rrc: 4 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x2aac6a50e5f9579 expref: -99 pid: 19867 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.879849:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764770.879850:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764770.879851:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764770.888050:0:11210:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764770.888058:0:11210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641c00/0x36f9bdc9e421c3c5 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x2/0x2 rrc: 4 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x2aac6a50e5f9579 expref: -99 pid: 19867 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.888064:0:11210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970641c00 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:1.0:1775764770.888068:0:11210:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641c00/0x36f9bdc9e421c3c5 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x2/0x2 rrc: 4 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x2aac6a50e5f9579 expref: -99 pid: 19867 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.888073:0:11210:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641c00/0x36f9bdc9e421c3c5 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x2/0x2 rrc: 4 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x2aac6a50e5f9579 expref: -99 pid: 19867 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.888079:0:11210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641c00/0x36f9bdc9e421c3c5 lrc: 3/0,0 mode: PR/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x2/0x2 rrc: 4 type: IBT gid 0 flags: 0x408400000000 nid: local remote: 0x2aac6a50e5f9579 expref: -99 pid: 19867 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.888166:0:11210:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641c00/0x36f9bdc9e421c3c5 lrc: 2/0,0 mode: --/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x2/0x2 rrc: 4 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x2aac6a50e5f9579 expref: -99 pid: 19867 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.888172:0:11210:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764770.888194:0:11210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641c00/0x36f9bdc9e421c3c5 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x2/0x2 rrc: 4 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x2aac6a50e5f9579 expref: -99 pid: 19867 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.888199:0:11210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641c00/0x36f9bdc9e421c3c5 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x2/0x2 rrc: 4 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x2aac6a50e5f9579 expref: -99 pid: 19867 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.888208:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764770.888209:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764770.888210:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764770.888215:0:2380:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1775764770.888226:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0a8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966927232:192.168.202.146@tcp:103:ldlm_bl.0 00000002:00010000:1.0:1775764770.891170:0:19869:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x281:0x0]) in obj [0x200000401:0x281:0x0], intent: readdir flags 0 00010000:00010000:1.0:1775764770.891176:0:19869:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/641 (0 0) 00010000:00010000:1.0:1775764770.891193:0:19869:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c3da lrc: 3/1,0 mode: --/PR res: [0x200000401:0x281:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 19869 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.891198:0:19869:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c3da lrc: 3/1,0 mode: --/PR res: [0x200000401:0x281:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19869 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.891204:0:19869:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c3da lrc: 3/1,0 mode: --/PR res: [0x200000401:0x281:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19869 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764770.891213:0:19869:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970b52d80 pname:cluuid:pid:xid:nid:opc:job ll_sa_19724:3f83546c-01a9-4156-8a97-44c446245d3e:19869:1862022966927616:192.168.202.146@tcp:101:ll_sa.0 00000100:00100000:1.0:1775764770.892305:0:19869:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f997130d540 going to sleep for 11 seconds 00000100:00100000:3.0:1775764770.892690:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966926976:192.168.202.146@tcp:36:dbench.0 00000100:00100000:1.0:1775764770.892866:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0a8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966927232:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764770.897197:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764770.897844:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978358400 00000100:00100000:1.0:1775764770.897851:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978358400 00000100:00100000:1.0:1775764770.897894:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24105 00000100:00100000:1.0:1775764770.897901:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0a8700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+5:11866:x1862022978358400:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764770.897912:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c3da lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x281:0x0].0x0 bits 0x2/0x2 rrc: 4 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x2aac6a50e5f95d4 expref: -99 pid: 19869 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.898048:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764770.898056:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0a8700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+5:11866:x1862022978358400:12345-192.168.202.146@tcp:104: Request processed in 154us (861us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764770.898063:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24105 00010000:00010000:3.0:1775764770.898102:0:5572:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1775764770.898109:0:5572:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c3da lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x281:0x0].0x0 bits 0x2/0x2 rrc: 4 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x2aac6a50e5f95d4 expref: -99 pid: 19869 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.898117:0:5572:0:(ldlm_lockd.c:1928:ldlm_handle_bl_callback()) Lock ffff8f9970640a00 is referenced, will be cancelled later 00010000:00010000:3.0:1775764770.898119:0:5572:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c3da lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x281:0x0].0x0 bits 0x2/0x2 rrc: 4 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x2aac6a50e5f95d4 expref: -99 pid: 19869 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.898126:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764770.898128:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764770.898130:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764770.903547:0:19869:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970f6b480 pname:cluuid:pid:xid:nid:opc:job ll_sa_19724:3f83546c-01a9-4156-8a97-44c446245d3e:19869:1862022966927872:192.168.202.146@tcp:37:ll_sa.0 00010000:00010000:3.0:1775764770.904429:0:19869:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c3da lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x281:0x0].0x0 bits 0x2/0x2 rrc: 4 type: IBT gid 0 flags: 0x10400400000000 nid: local remote: 0x2aac6a50e5f95d4 expref: -99 pid: 19869 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.904439:0:19869:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c3da lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x281:0x0].0x0 bits 0x2/0x2 rrc: 4 type: IBT gid 0 flags: 0x10400400000000 nid: local remote: 0x2aac6a50e5f95d4 expref: -99 pid: 19869 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.904463:0:19869:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00010000:00010000:3.0:1775764770.904506:0:19869:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c3e8 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x281:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 19869 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.904514:0:19869:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c3e8 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x281:0x0].0x0 bits 0x3/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19869 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.904542:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764770.904564:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c3da lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x281:0x0].0x0 bits 0x2/0x2 rrc: 5 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x2aac6a50e5f95d4 expref: -99 pid: 19869 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.904572:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970640a00 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:1.0:1775764770.904591:0:19801:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c3da lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x281:0x0].0x0 bits 0x2/0x2 rrc: 5 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x2aac6a50e5f95d4 expref: -99 pid: 19869 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.904600:0:19801:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c3da lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x281:0x0].0x0 bits 0x2/0x2 rrc: 5 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x2aac6a50e5f95d4 expref: -99 pid: 19869 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.904635:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c3da lrc: 3/0,0 mode: PR/PR res: [0x200000401:0x281:0x0].0x0 bits 0x2/0x2 rrc: 5 type: IBT gid 0 flags: 0x408400000000 nid: local remote: 0x2aac6a50e5f95d4 expref: -99 pid: 19869 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.904699:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c3da lrc: 2/0,0 mode: --/PR res: [0x200000401:0x281:0x0].0x0 bits 0x2/0x2 rrc: 5 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x2aac6a50e5f95d4 expref: -99 pid: 19869 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.904707:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764770.904736:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c3da lrc: 1/0,0 mode: --/PR res: [0x200000401:0x281:0x0].0x0 bits 0x2/0x2 rrc: 5 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x2aac6a50e5f95d4 expref: -99 pid: 19869 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.904778:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c3da lrc: 0/0,0 mode: --/PR res: [0x200000401:0x281:0x0].0x0 bits 0x2/0x2 rrc: 5 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x2aac6a50e5f95d4 expref: -99 pid: 19869 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.904787:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764770.904788:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764770.904790:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764770.904798:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1775764770.904805:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970b52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966928256:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1775764770.908839:0:2381:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c3e8 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x281:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9613 expref: -99 pid: 19869 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.908946:0:2381:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c3e8 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x282:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9613 expref: -99 pid: 19869 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.908955:0:2381:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c3e8 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x282:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9613 expref: -99 pid: 19869 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.908971:0:2381:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c3e8 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x282:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9613 expref: -99 pid: 19869 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.908982:0:2381:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c3e8 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x282:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9613 expref: -99 pid: 19869 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.908994:0:2381:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c3e8 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x282:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9613 expref: -99 pid: 19869 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.909003:0:2381:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c3e8 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x282:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9613 expref: -99 pid: 19869 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.909010:0:2381:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c3e8 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x282:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9613 expref: -99 pid: 19869 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764770.909033:0:2381:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970ede680 x1862022966928128/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 672/968 e 0 to 0 dl 1775764786 ref 1 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:3.0:1775764770.909045:0:2381:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c3e8 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x282:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9613 expref: -99 pid: 19869 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:3.0:1775764770.909062:0:2381:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c3e8 from it ffff8f9966932588 00010000:00010000:3.0:1775764770.909065:0:2381:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c3e8 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x282:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9613 expref: -99 pid: 19869 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.909072:0:2381:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c3e8 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x282:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9613 expref: -99 pid: 19869 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000100:00100000:3.0:1775764770.909102:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966928128:192.168.202.146@tcp:63:ll_sa.0 00010000:00010000:3.0:1775764770.909131:0:19780:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c3e8 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x282:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9613 expref: -99 pid: 19869 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764770.909211:0:19780:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x282:0x0](ffff8f9952db8908): rc = 0 00000080:00200000:3.0:1775764770.909228:0:19780:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x282:0x0]: layout version change: 3 -> 3 00000080:00010000:3.0:1775764770.909238:0:19780:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x282:0x0](ffff8f9952db8908) for lock 0x36f9bdc9e421c3e8 00000080:00010000:3.0:1775764770.909248:0:19780:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c3e8 from it ffff8f9966932588 00010000:00010000:3.0:1775764770.909251:0:19780:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c3e8 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x282:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9613 expref: -99 pid: 19869 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764770.909258:0:19780:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c3e8 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x282:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9613 expref: -99 pid: 19869 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764770.909295:0:19870:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x282:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764770.909305:0:19870:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x282:0x0] 00000008:00010000:3.0:1775764770.909329:0:19870:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:3.0:1775764770.909332:0:19870:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d5630, osc lock ffff8f9966ace4c0, flags 0x400041012 00010000:00010000:3.0:1775764770.909349:0:19870:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000004f97dc86/0x36f9bdc9e41ca30d lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x3c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ed287 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 107 lru_type: 1 00010000:00010000:3.0:1775764770.909363:0:19870:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000004f97dc86/0x36f9bdc9e41ca30d lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x3c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ed287 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 107 lru_type: 1 00010000:00010000:3.0:1775764770.909372:0:19870:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000004f97dc86/0x36f9bdc9e41ca30d lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x3c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ed287 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 107 lru_type: 1 00010000:00010000:3.0:1775764770.909379:0:19870:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000004f97dc86/0x36f9bdc9e41ca30d lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x3c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ed287 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 108 lru_type: 1 00000020:00010000:3.0:1775764770.909389:0:19870:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef50000 (ffff8f9970314b48/0) at cl_lock_release():245 00000020:00010000:3.0:1775764770.909392:0:19870:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef50000 (ffff8f9970314b48/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.909395:0:19870:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d5630 (ffff8f9970314e18/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.909398:0:19870:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef50000 (ffff8f9970314b48/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764770.909401:0:19870:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d5630 (ffff8f9970314b48/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764770.909416:0:19870:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x282:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1775764770.910908:0:11138:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1775764770.910914:0:11138:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99200/0x36f9bdc9e421c3e1 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f95f7 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.910922:0:11138:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970a99200 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:3.0:1775764770.910927:0:11138:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99200/0x36f9bdc9e421c3e1 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b7:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f95f7 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.910936:0:11138:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99200/0x36f9bdc9e421c3e1 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b7:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f95f7 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.911012:0:11138:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99200/0x36f9bdc9e421c3e1 lrc: 3/0,0 mode: PR/PR res: [0x200000401:0x2b7:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f95f7 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.911027:0:11138:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966c08400/0x36f9bdc9e4218ade lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5eda08 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1775764770.911057:0:11138:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99200/0x36f9bdc9e421c3e1 lrc: 2/0,0 mode: --/PR res: [0x200000401:0x2b7:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f95f7 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.911064:0:11138:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764770.911092:0:11138:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99200/0x36f9bdc9e421c3e1 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2b7:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f95f7 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.911099:0:11138:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99200/0x36f9bdc9e421c3e1 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2b7:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f95f7 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.911109:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764770.911111:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764770.911113:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764770.911120:0:2381:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1775764770.911134:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966928384:192.168.202.146@tcp:103:ldlm_bl.0 00000002:00010000:1.0:1775764770.911491:0:19871:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2b2:0x0]) in obj [0x200000401:0x2b2:0x0], intent: readdir flags 0 00010000:00010000:1.0:1775764770.911515:0:19871:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/690 (0 0) 00010000:00010000:1.0:1775764770.911550:0:19871:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c3ef lrc: 3/1,0 mode: --/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 19871 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.911576:0:19871:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c3ef lrc: 3/1,0 mode: --/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19871 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.911586:0:19871:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c3ef lrc: 3/1,0 mode: --/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19871 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764770.911610:0:19871:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970b51f80 pname:cluuid:pid:xid:nid:opc:job ll_sa_19723:3f83546c-01a9-4156-8a97-44c446245d3e:19871:1862022966928768:192.168.202.146@tcp:101:ll_sa.0 00000100:00100000:1.0:1775764770.911673:0:19871:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f997130d0c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764770.915392:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970b52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966928256:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764770.916129:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764770.916171:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978358528 00000100:00100000:1.0:1775764770.916179:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978358528 00000100:00100000:1.0:1775764770.916225:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24106 00000100:00100000:1.0:1775764770.916234:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0a8e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+5:16259:x1862022978358528:12345-192.168.202.146@tcp:105: 00000100:00100000:3.0:1775764770.916244:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966928384:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1775764770.916248:0:5561:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c3ef lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f963d expref: -99 pid: 19871 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.916260:0:5561:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c3ef lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f963d expref: -99 pid: 19871 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.916269:0:5561:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c3ef lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f963d expref: -99 pid: 19871 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.916278:0:5561:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c3ef lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x10080000000000 nid: local remote: 0x2aac6a50e5f963d expref: -99 pid: 19871 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.916287:0:5561:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c3ef lrc: 5/1,0 mode: PR/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x10080000000000 nid: local remote: 0x2aac6a50e5f963d expref: -99 pid: 19871 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.916296:0:5561:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c3ef lrc: 5/1,0 mode: PR/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x80000000000 nid: local remote: 0x2aac6a50e5f963d expref: -99 pid: 19871 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.916318:0:5561:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8f9970640200) 00000100:00100000:1.0:1775764770.916362:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0a8e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+5:16259:x1862022978358528:12345-192.168.202.146@tcp:105: Request processed in 129us (235us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764770.916370:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24106 00000100:00100000:0.0:1775764770.916488:0:19871:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970b51f80 pname:cluuid:pid:xid:nid:opc:job ll_sa_19723:3f83546c-01a9-4156-8a97-44c446245d3e:19871:1862022966928768:192.168.202.146@tcp:101:ll_sa.0 00010000:00010000:0.0:1775764770.916504:0:19871:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c3ef lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f963d expref: -99 pid: 19871 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764770.916518:0:19871:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=4 disposition=0, status=0 req@ffff8f9970b51f80 x1862022966928768/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 328/344 e 0 to 0 dl 1775764786 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1775764770.917301:0:19871:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0a9f80 pname:cluuid:pid:xid:nid:opc:job ll_sa_19723:3f83546c-01a9-4156-8a97-44c446245d3e:19871:1862022966928896:192.168.202.146@tcp:37:ll_sa.0 00000100:00100000:0.0:1775764770.917371:0:19871:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16ff00 going to sleep for 11 seconds 00000100:00100000:3.0:1775764770.920177:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970f6b100 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966928640:192.168.202.146@tcp:36:dbench.0 00000080:00200000:3.0:1775764770.920242:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2b7:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1775764770.920250:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764770.920263:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:3.0:1775764770.920265:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:3.0:1775764770.920279:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2b7:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764770.920629:0:19723:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2b7:0x0](ffff8f9952d8f448) 00000008:00010000:3.0:1775764770.920654:0:19723:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 1 [cma]time 1775764055 1775764055 0 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c6478f03/0x36f9bdc9e41caa8b lrc: 1/0,0 mode: PW/PW res: [0x240000403:0x53:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee630 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 1 00000080:00200000:3.0:1775764770.920735:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.920763:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.920769:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:3.0:1775764770.920775:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.920780:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:3.0:1775764770.920783:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.920786:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:3.0:1775764770.920789:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.920792:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PM, flags=17 00000080:00200000:3.0:1775764770.920794:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b2:0x0](ffff8f9952d8ba88), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.920805:0:19723:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=T1.XLS, dir=[0x200000401:0x2b2:0x0](ffff8f9952d8ba88), flags=1 00000080:00200000:3.0:1775764770.920810:0:19723:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=T1.XLS, dir=[0x200000401:0x2b2:0x0](ffff8f9952d8ba88), intent=getattr 00000002:00010000:3.0:1775764770.920846:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: T1.XLS,[0x0:0x0:0x0]) in obj [0x200000401:0x2b2:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764770.920896:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3f6 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b2:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.920904:0:19723:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3f6 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b2:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.920913:0:19723:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3f6 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b2:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764770.920933:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966929024:192.168.202.146@tcp:101:dbench.0 00000100:00100000:3.0:1775764770.921380:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99707ef540 going to sleep for 11 seconds 00000100:00100000:3.0:1775764770.925728:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966929024:192.168.202.146@tcp:101:dbench.0 00010000:00010000:3.0:1775764770.925786:0:19723:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3f6 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2b2:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9675 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.925804:0:19723:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3f6 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2b4:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9675 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.925812:0:19723:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3f6 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2b4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9675 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.925822:0:19723:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3f6 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2b4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9675 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.925833:0:19723:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3f6 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2b4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9675 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.925845:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3f6 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2b4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9675 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.925852:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3f6 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2b4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9675 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.925859:0:19723:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3f6 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2b4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9675 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764770.925871:0:19723:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970ede680 x1862022966929024/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/928 e 0 to 0 dl 1775764786 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'dbench.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1775764770.925883:0:19723:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3f6 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9675 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764770.926909:0:19871:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0a9f80 pname:cluuid:pid:xid:nid:opc:job ll_sa_19723:3f83546c-01a9-4156-8a97-44c446245d3e:19871:1862022966928896:192.168.202.146@tcp:37:ll_sa.0 00000002:00010000:0.0:1775764770.927082:0:19723:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3f6 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9675 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.927094:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421849b lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b4:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5ec0af expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.927104:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421849b lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5ec0af expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.927111:0:19723:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3f6 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9675 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.927116:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3f6 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x2aac6a50e5f9675 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.927123:0:19723:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3f6 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x2aac6a50e5f9675 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.927796:0:19871:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c3ef lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f963d expref: -99 pid: 19871 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.927804:0:19871:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c3ef lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f963d expref: -99 pid: 19871 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764770.927857:0:19871:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c3fd lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b2:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 19871 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.927864:0:19871:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c3fd lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b2:0x0].0x0 bits 0x3/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19871 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.928245:0:19723:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000080:00200000:0.0:1775764770.928254:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2b2:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1775764770.928334:0:19723:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x2b4:0x0](ffff8f9952d8a1c8): rc = 0 00000080:00200000:0.0:1775764770.928354:0:19723:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x2b4:0x0]: layout version change: 3 -> 3 00000080:00010000:0.0:1775764770.928362:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2b4:0x0](ffff8f9952d8a1c8) for lock 0x36f9bdc9e421849b 00000080:00010000:0.0:1775764770.928372:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2b4:0x0](ffff8f9952d8a1c8) 00000080:00010000:0.0:1775764770.928375:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421849b from it ffffafbac1c7bb20 00010000:00010000:0.0:1775764770.928378:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421849b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5ec0af expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.928399:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421849b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5ec0af expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200008:0.0:1775764770.928435:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file T1.XLS:[0x200000401:0x2b4:0x0](ffff8f9952d8a1c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764770.928441:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2b4:0x0](ffff8f9952d8a1c8),name=T1.XLS 00000002:00010000:0.0:1775764770.928452:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2b4:0x0]) in obj [0x200000401:0x2b2:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764770.928461:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421849b lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b4:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5ec0af expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764770.928470:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421849b lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5ec0af expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764770.928478:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2b2:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764770.928482:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2b4:0x0](ffff8f9952d8a1c8) 00000080:00010000:0.0:1775764770.928485:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2b4:0x0](ffff8f9952d8a1c8) for lock 0x36f9bdc9e421849b 00000080:00010000:0.0:1775764770.928488:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421849b from it ffffafbac1c7bcb8 00010000:00010000:0.0:1775764770.928491:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421849b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5ec0af expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764770.928503:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421849b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5ec0af expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1775764770.928532:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2b4:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1775764770.928542:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2b4:0x0] 00000008:00010000:0.0:1775764770.928570:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1775764770.928574:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0df9b0, osc lock ffff8f996686f430, flags 0x400001002 00010000:00010000:0.0:1775764770.928591:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000719d22c5/0x36f9bdc9e41caa30 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x51:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee550 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 21 lru_type: 1 00010000:00010000:0.0:1775764770.928601:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000719d22c5/0x36f9bdc9e41caa30 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x51:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee550 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 21 lru_type: 1 00000020:00200000:0.0:1775764770.928630:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1775764770.928634:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000719d22c5/0x36f9bdc9e41caa30 lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x51:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ee550 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 21 lru_type: 1 00010000:00010000:0.0:1775764770.928642:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000719d22c5/0x36f9bdc9e41caa30 lrc: 4/0,0 mode: PW/PW res: [0x280000403:0x51:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee550 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 22 lru_type: 1 00000020:00200000:0.0:1775764770.928651:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1775764770.928660:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2b4:0x0] updating i_size 260096 i_blocks 1 00000020:00010000:0.0:1775764770.928663:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_release():245 00000020:00010000:0.0:1775764770.928666:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.928668:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0df9b0 (ffff8f9966dc2128/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.928672:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764770.928675:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0df9b0 (ffff8f9966dc2368/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764770.928690:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2b4:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764770.928718:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.928722:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.928727:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:0.0:1775764770.928730:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.928733:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=16 00000080:00200000:0.0:1775764770.928756:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.928760:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:0.0:1775764770.928763:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.928767:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PM, flags=16 00000080:00200000:0.0:1775764770.928770:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b2:0x0](ffff8f9952d8ba88), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.928775:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=T1.XLS, flags=0 00000080:00200000:0.0:1775764770.928779:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b2:0x0](ffff8f9952d8ba88), inode mode 41c0 mask 3 00000080:00200000:0.0:1775764770.928783:0:19723:0:(namei.c:2395:ll_unlink()) VFS Op:name=T1.XLS, dir=[0x200000401:0x2b2:0x0](ffff8f9952d8ba88) 00010000:00010000:0.0:1775764770.928815:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c3ef lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x2/0x0 rrc: 6 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f963d expref: -99 pid: 19871 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.928837:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3f6 lrc: 3/0,0 mode: PR/PR res: [0x200000401:0x2b4:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f9675 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1775764770.928852:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2b4:0x0]: losing layout lock 00010000:00010000:0.0:1775764770.928866:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421849b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b4:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5ec0af expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1775764770.928879:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2b4:0x0]: losing layout lock 00010000:00010000:0.0:1775764770.928903:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3f6 lrc: 2/0,0 mode: --/PR res: [0x200000401:0x2b4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9675 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.928914:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421849b lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2b4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5ec0af expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764770.928920:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c3ef lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f963d expref: -99 pid: 19871 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.928925:0:19723:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:0.0:1775764770.928928:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421849b lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2b4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5ec0af expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764770.928940:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c3ef lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f963d expref: -99 pid: 19871 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000020:00100000:0.0:1775764770.929094:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1775764770.929104:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f0a9500 x1862022966929792/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764770.929122:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0a9500 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966929792:192.168.202.146@tcp:36:dbench.0 00000100:00100000:0.0:1775764770.929167:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f3c0 going to sleep for 11 seconds 00010000:00010000:1.0:1775764770.929468:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764770.929475:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3f6 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2b4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9675 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.929483:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99435d4800 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:1.0:1775764770.929487:0:5571:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3f6 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2b4:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9675 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.929495:0:5571:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3f6 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2b4:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9675 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.929501:0:5571:0:(ldlm_request.c:1765:ldlm_cli_cancel()) ### lock is already being destroyed 00010000:00010000:1.0:1775764770.929504:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3f6 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2b4:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9675 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.929511:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c3f6 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2b4:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9675 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764770.929528:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764770.929530:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764770.929532:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1775764770.939064:0:19872:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2b9:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764770.939075:0:19872:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2b9:0x0] 00000008:00010000:1.0:1775764770.939096:0:19872:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:1.0:1775764770.939099:0:19872:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f1c7130, osc lock ffff8f995875db50, flags 0x400041012 00010000:00010000:1.0:1775764770.939395:0:19872:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000d1d1380c/0x36f9bdc9e41caadf lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x53:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee748 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 26 lru_type: 1 00010000:00010000:1.0:1775764770.939412:0:19872:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000d1d1380c/0x36f9bdc9e41caadf lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x53:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee748 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 26 lru_type: 1 00010000:00010000:1.0:1775764770.939422:0:19872:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000d1d1380c/0x36f9bdc9e41caadf lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x53:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ee748 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 26 lru_type: 1 00010000:00010000:1.0:1775764770.939430:0:19872:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000d1d1380c/0x36f9bdc9e41caadf lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x53:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee748 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 27 lru_type: 1 00000020:00010000:1.0:1775764770.939444:0:19872:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_release():245 00000020:00010000:1.0:1775764770.939448:0:19872:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764770.939451:0:19872:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f1c7130 (ffff8f99705b8ea8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764770.939455:0:19872:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764770.939457:0:19872:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f1c7130 (ffff8f99705b8128/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764770.939481:0:19872:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2b9:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:3.0:1775764770.940511:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970f69500 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966929408:192.168.202.146@tcp:37:dbench.0 00000080:00200000:0.0:1775764770.941067:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2b4:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764770.941077:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1775764770.941093:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:0.0:1775764770.941094:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:0.0:1775764770.941121:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2b4:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1775764770.941151:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bec00/0x36f9bdc9e421c244 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x278:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b38 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1775764770.941162:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bec00/0x36f9bdc9e421c244 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x278:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b38 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:3.0:1775764770.941225:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.941230:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.941235:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:3.0:1775764770.941240:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.941243:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:3.0:1775764770.941246:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.941249:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:3.0:1775764770.941251:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.941254:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=WORD, flags=17 00000080:00200000:3.0:1775764770.941256:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x278:0x0](ffff8f9952dbf448), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.941266:0:19724:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=FACTS.DOC, dir=[0x200000401:0x278:0x0](ffff8f9952dbf448), flags=1 00000080:00200000:3.0:1775764770.941270:0:19724:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=FACTS.DOC, dir=[0x200000401:0x278:0x0](ffff8f9952dbf448), intent=getattr 00000002:00010000:3.0:1775764770.941288:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: FACTS.DOC,[0x0:0x0:0x0]) in obj [0x200000401:0x278:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764770.941332:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c404 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x278:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.941340:0:19724:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c404 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x278:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.941349:0:19724:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c404 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x278:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764770.941368:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966929920:192.168.202.146@tcp:101:dbench.0 00000100:00100000:3.0:1775764770.941448:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99707ef6c0 going to sleep for 11 seconds 00000080:00200000:0.0:1775764770.941480:0:19723:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2b4:0x0](ffff8f9952d8a1c8) 00000008:00010000:0.0:1775764770.942016:0:19723:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 1 [cma]time 1775764055 1775764055 0 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000719d22c5/0x36f9bdc9e41caa30 lrc: 1/0,0 mode: PW/PW res: [0x280000403:0x51:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee550 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 22 lru_type: 1 00000080:00200000:0.0:1775764770.942094:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.942100:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.942104:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:0.0:1775764770.942108:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.942111:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:0.0:1775764770.942114:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.942117:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:0.0:1775764770.942119:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.942122:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PM, flags=17 00000080:00200000:0.0:1775764770.942124:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b2:0x0](ffff8f9952d8ba88), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.942133:0:19723:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=MOVED.DOC, dir=[0x200000401:0x2b2:0x0](ffff8f9952d8ba88), flags=1 00000080:00200000:0.0:1775764770.942136:0:19723:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=MOVED.DOC, dir=[0x200000401:0x2b2:0x0](ffff8f9952d8ba88), intent=getattr 00010000:00010000:0.0:1775764770.942149:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c3fd lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f96bb expref: -99 pid: 19871 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:0.0:1775764770.942162:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c3fd from it ffffafbac1c7b900 00010000:00010000:0.0:1775764770.942166:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c3fd lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f96bb expref: -99 pid: 19871 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764770.942173:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c3fd lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f96bb expref: -99 pid: 19871 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:0.0:1775764770.942197:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file MOVED.DOC:[0x200000401:0x2b9:0x0](ffff8f9952d11148), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764770.942203:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2b9:0x0](ffff8f9952d11148),name=MOVED.DOC 00000002:00010000:0.0:1775764770.942213:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2b9:0x0]) in obj [0x200000401:0x2b2:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764770.942525:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971079600/0x36f9bdc9e4218be8 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5edd34 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.942536:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971079600/0x36f9bdc9e4218be8 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5edd34 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764770.942543:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2b2:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764770.942549:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2b9:0x0](ffff8f9952d11148) 00000080:00010000:0.0:1775764770.942552:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2b9:0x0](ffff8f9952d11148) for lock 0x36f9bdc9e4218be8 00000080:00010000:0.0:1775764770.942558:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e4218be8 from it ffffafbac1c7bcb8 00010000:00010000:0.0:1775764770.942560:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971079600/0x36f9bdc9e4218be8 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5edd34 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.942567:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971079600/0x36f9bdc9e4218be8 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5edd34 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764770.942593:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2b9:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1775764770.942601:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2b9:0x0] 00000008:00010000:0.0:1775764770.942632:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1775764770.942636:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0df9b0, osc lock ffff8f996686f430, flags 0x400001002 00010000:00010000:0.0:1775764770.942651:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000d1d1380c/0x36f9bdc9e41caadf lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x53:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee748 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 27 lru_type: 1 00010000:00010000:0.0:1775764770.942662:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000d1d1380c/0x36f9bdc9e41caadf lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x53:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee748 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 27 lru_type: 1 00000020:00200000:0.0:1775764770.942672:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1775764770.942676:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000d1d1380c/0x36f9bdc9e41caadf lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x53:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ee748 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 27 lru_type: 1 00010000:00010000:0.0:1775764770.942689:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000d1d1380c/0x36f9bdc9e41caadf lrc: 4/0,0 mode: PW/PW res: [0x280000403:0x53:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee748 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 28 lru_type: 1 00000020:00200000:0.0:1775764770.942698:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1775764770.942706:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2b9:0x0] updating i_size 260096 i_blocks 1 00000020:00010000:0.0:1775764770.942710:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_release():245 00000020:00010000:0.0:1775764770.942712:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.942715:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0df9b0 (ffff8f9966dc2128/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.942719:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764770.942722:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0df9b0 (ffff8f9966dc2368/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764770.942769:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2b9:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764770.942794:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.942799:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.942802:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:0.0:1775764770.942806:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.942809:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=16 00000080:00200000:0.0:1775764770.942811:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.942814:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:0.0:1775764770.942816:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.942819:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PM, flags=16 00000080:00200000:0.0:1775764770.942821:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b2:0x0](ffff8f9952d8ba88), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.942825:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=MOVED.DOC, flags=0 00000080:00200000:0.0:1775764770.942829:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b2:0x0](ffff8f9952d8ba88), inode mode 41c0 mask 3 00000080:00200000:0.0:1775764770.942833:0:19723:0:(namei.c:2395:ll_unlink()) VFS Op:name=MOVED.DOC, dir=[0x200000401:0x2b2:0x0](ffff8f9952d8ba88) 00010000:00010000:0.0:1775764770.942872:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c3fd lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f96bb expref: -99 pid: 19871 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1775764770.942899:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2b9:0x0]: losing layout lock 00010000:00010000:0.0:1775764770.942918:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971079600/0x36f9bdc9e4218be8 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5edd34 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764770.942999:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2b9:0x0]: losing layout lock 00010000:00010000:0.0:1775764770.943031:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c3fd lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2b9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f96bb expref: -99 pid: 19871 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764770.943039:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971079600/0x36f9bdc9e4218be8 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2b9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5edd34 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764770.943045:0:19723:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1775764770.943048:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c3fd lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2b9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f96bb expref: -99 pid: 19871 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764770.943059:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971079600/0x36f9bdc9e4218be8 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2b9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5edd34 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000020:00100000:0.0:1775764770.943079:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1775764770.943088:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f0abb80 x1862022966930176/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764770.943107:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0abb80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966930176:192.168.202.146@tcp:36:dbench.0 00000100:00100000:0.0:1775764770.943164:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f240 going to sleep for 11 seconds 00000100:00100000:3.0:1775764770.946472:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966929920:192.168.202.146@tcp:101:dbench.0 00010000:00010000:3.0:1775764770.946494:0:19724:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c404 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x278:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f96d7 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.946509:0:19724:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c404 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x27e:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f96d7 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.946516:0:19724:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c404 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x27e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f96d7 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.946526:0:19724:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c404 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x27e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f96d7 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.946536:0:19724:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c404 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x27e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f96d7 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.946547:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c404 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x27e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f96d7 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.946554:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c404 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x27e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f96d7 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.946561:0:19724:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c404 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x27e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f96d7 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764770.946572:0:19724:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970edca80 x1862022966929920/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/928 e 0 to 0 dl 1775764786 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'dbench.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1775764770.946582:0:19724:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c404 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x27e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f96d7 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:3.0:1775764770.946593:0:19724:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c404 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x27e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f96d7 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.946605:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99700e5000/0x36f9bdc9e4219c50 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x27e:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f16f4 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764770.946713:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99700e5000/0x36f9bdc9e4219c50 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x27e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f16f4 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764770.946722:0:19724:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c404 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x27e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f96d7 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.946728:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c404 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x27e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x2aac6a50e5f96d7 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.946735:0:19724:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c404 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x27e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x2aac6a50e5f96d7 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764770.946819:0:19724:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000080:00200000:3.0:1775764770.946826:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x278:0x0] suppgids 0 -1: rc 0 00000080:00200000:3.0:1775764770.946909:0:19724:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x27e:0x0](ffff8f9952dbe3c8): rc = 0 00000080:00200000:3.0:1775764770.946930:0:19724:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x27e:0x0]: layout version change: 3 -> 3 00000080:00010000:3.0:1775764770.946940:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x27e:0x0](ffff8f9952dbe3c8) for lock 0x36f9bdc9e4219c50 00000080:00010000:3.0:1775764770.947017:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x27e:0x0](ffff8f9952dbe3c8) 00000080:00010000:3.0:1775764770.947020:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e4219c50 from it ffffafbac1c5bb20 00010000:00010000:3.0:1775764770.947024:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99700e5000/0x36f9bdc9e4219c50 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x27e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f16f4 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764770.947032:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99700e5000/0x36f9bdc9e4219c50 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x27e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f16f4 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200008:3.0:1775764770.947082:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file FACTS.DOC:[0x200000401:0x27e:0x0](ffff8f9952dbe3c8), request_mask 2047, flags 0, foreign 0 00010000:00010000:1.0:1775764770.949165:0:19873:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8c000/0x36f9bdc9e4219a89 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x27c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f111a expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764770.949182:0:19873:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8c000/0x36f9bdc9e4219a89 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x27c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f111a expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00010000:1.0:1775764770.949189:0:19873:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e4219a89 from it ffffafbac1f27d48 00010000:00010000:1.0:1775764770.949194:0:19873:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8c000/0x36f9bdc9e4219a89 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x27c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f111a expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764770.949201:0:19873:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8c000/0x36f9bdc9e4219a89 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x27c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f111a expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00000080:00200000:3.0:1775764770.949300:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x27e:0x0](ffff8f9952dbe3c8),name=FACTS.DOC 00000002:00010000:3.0:1775764770.949313:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x27e:0x0]) in obj [0x200000401:0x278:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764770.949320:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99700e5000/0x36f9bdc9e4219c50 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x27e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f16f4 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764770.949326:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99700e5000/0x36f9bdc9e4219c50 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x27e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f16f4 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764770.949331:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x278:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764770.949336:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x27e:0x0](ffff8f9952dbe3c8) 00000080:00010000:3.0:1775764770.949338:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x27e:0x0](ffff8f9952dbe3c8) for lock 0x36f9bdc9e4219c50 00000080:00010000:3.0:1775764770.949342:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e4219c50 from it ffffafbac1c5bcb8 00010000:00010000:3.0:1775764770.949344:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99700e5000/0x36f9bdc9e4219c50 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x27e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f16f4 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764770.949348:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99700e5000/0x36f9bdc9e4219c50 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x27e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f16f4 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1775764770.949381:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x27e:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764770.949388:0:19724:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x27e:0x0] 00000008:00010000:3.0:1775764770.949409:0:19724:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1775764770.949412:0:19724:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d5cb0, osc lock ffff8f9966ace4c0, flags 0x400001002 00010000:00010000:3.0:1775764770.949425:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000024570130/0x36f9bdc9e41ca265 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x3b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ed08f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 0 00010000:00010000:3.0:1775764770.949433:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000024570130/0x36f9bdc9e41ca265 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x3b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ed08f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 0 00000020:00200000:3.0:1775764770.949445:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1775764770.949447:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000024570130/0x36f9bdc9e41ca265 lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x3b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ed08f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 0 00010000:00010000:3.0:1775764770.949453:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000024570130/0x36f9bdc9e41ca265 lrc: 4/0,0 mode: PW/PW res: [0x280000403:0x3b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ed08f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00000020:00200000:3.0:1775764770.949459:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1775764770.949465:0:19724:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x27e:0x0] updating i_size 260096 i_blocks 1 00000020:00010000:3.0:1775764770.949467:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef50000 (ffff8f9970314b48/0) at cl_lock_release():245 00000020:00010000:3.0:1775764770.949469:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef50000 (ffff8f9970314b48/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.949471:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d5cb0 (ffff8f9970314e18/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.949474:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef50000 (ffff8f9970314b48/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764770.949475:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d5cb0 (ffff8f9970314b48/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764770.949495:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x27e:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764770.949525:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.949528:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.949531:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:3.0:1775764770.949535:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.949537:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=16 00000080:00200000:3.0:1775764770.949539:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.949541:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:3.0:1775764770.949543:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.949545:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=WORD, flags=16 00000080:00200000:3.0:1775764770.949547:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x278:0x0](ffff8f9952dbf448), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.949551:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=FACTS.DOC, flags=0 00000080:00200000:3.0:1775764770.949558:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x278:0x0](ffff8f9952dbf448), inode mode 41c0 mask 3 00000080:00200000:3.0:1775764770.949561:0:19724:0:(namei.c:2395:ll_unlink()) VFS Op:name=FACTS.DOC, dir=[0x200000401:0x278:0x0](ffff8f9952dbf448) 00010000:00010000:3.0:1775764770.949585:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bec00/0x36f9bdc9e421c244 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x278:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f8b38 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:3.0:1775764770.949596:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5e00/0x36f9bdc9e41ca1a1 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x278:0x0].0x0 bits 0x5/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ece9e expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 80 lru_type: 1 00010000:00010000:3.0:1775764770.949602:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5800/0x36f9bdc9e41ca1e7 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x278:0x0].0x0 bits 0x11/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ecf62 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00010000:00010000:3.0:1775764770.949669:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99700e5000/0x36f9bdc9e4219c50 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x27e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f16f4 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1775764770.949688:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x27e:0x0]: losing layout lock 00010000:00010000:3.0:1775764770.949723:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99700e5000/0x36f9bdc9e4219c50 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x27e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f16f4 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1775764770.949731:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bec00/0x36f9bdc9e421c244 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x278:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8b38 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:3.0:1775764770.949753:0:19724:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1775764770.949757:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99700e5000/0x36f9bdc9e4219c50 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x27e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f16f4 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1775764770.949773:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bec00/0x36f9bdc9e421c244 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x278:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8b38 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000020:00100000:3.0:1775764770.949789:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 2 is allocated opc 36, max 7 00000100:00100000:3.0:1775764770.949800:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970edca80 x1862022966930560/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764770.949822:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966930560:192.168.202.146@tcp:36:dbench.0 00000100:00100000:3.0:1775764770.949907:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99707ef180 going to sleep for 11 seconds 00000080:00200000:0.0:1775764770.951368:0:19874:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x27c:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1775764770.951374:0:19874:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x27c:0x0] 00000008:00010000:0.0:1775764770.951406:0:19874:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:0.0:1775764770.951410:0:19874:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0df9b0, osc lock ffff8f996686f430, flags 0x400041012 00010000:00010000:0.0:1775764770.951429:0:19874:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000bb0f9183/0x36f9bdc9e41ca21f lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x3a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ecffc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 0 00010000:00010000:0.0:1775764770.951442:0:19874:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000bb0f9183/0x36f9bdc9e41ca21f lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x3a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ecffc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 0 00010000:00010000:0.0:1775764770.951452:0:19874:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000bb0f9183/0x36f9bdc9e41ca21f lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x3a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ecffc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 0 00010000:00010000:0.0:1775764770.951461:0:19874:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000bb0f9183/0x36f9bdc9e41ca21f lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x3a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ecffc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 1 00000020:00010000:0.0:1775764770.951483:0:19874:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_release():245 00000020:00010000:0.0:1775764770.951486:0:19874:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.951489:0:19874:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0df9b0 (ffff8f9966dc2128/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.951493:0:19874:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764770.951495:0:19874:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0df9b0 (ffff8f9966dc2368/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764770.951510:0:19874:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x27c:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:3.0:1775764770.951698:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970f68e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966930304:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1775764770.951821:0:19873:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c40b lrc: 3/1,0 mode: --/CR res: [0x200000401:0x278:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 19873 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.951829:0:19873:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c40b lrc: 3/1,0 mode: --/CR res: [0x200000401:0x278:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19873 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.951849:0:19873:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c412 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x278:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 19873 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.951854:0:19873:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c412 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x278:0x0].0x0 bits 0x3/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19873 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.951867:0:19873:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd600/0x36f9bdc9e421c419 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x278:0x0].0x0 bits 0x0/0x0 rrc: 6 type: IBT flags: 0x10000000000000 pid: 19873 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.951872:0:19873:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd600/0x36f9bdc9e421c419 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x278:0x0].0x0 bits 0x3/0x0 rrc: 6 type: IBT flags: 0x0 pid: 19873 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764770.951926:0:2380:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1775764770.951938:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0a9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966930688:192.168.202.146@tcp:63:ll_sa.0 00000100:00100000:3.0:1775764770.956243:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966930560:192.168.202.146@tcp:36:dbench.0 00000080:00200000:3.0:1775764770.956319:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x27e:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1775764770.956332:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764770.956357:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:3.0:1775764770.956359:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:3.0:1775764770.956402:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x27e:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764770.957019:0:19724:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x27e:0x0](ffff8f9952dbe3c8) 00000008:00010000:3.0:1775764770.957059:0:19724:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 1 [cma]time 1775764051 1775764051 0 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000024570130/0x36f9bdc9e41ca265 lrc: 1/0,0 mode: PW/PW res: [0x280000403:0x3b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ed08f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00000080:00200000:3.0:1775764770.957141:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.957147:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.957152:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:3.0:1775764770.957156:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.957158:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:3.0:1775764770.957160:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.957162:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:3.0:1775764770.957164:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.957166:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=WORD, flags=17 00000080:00200000:3.0:1775764770.957168:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x278:0x0](ffff8f9952dbf448), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.957169:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=BASEMACH.DOC, flags=1 00010000:00010000:3.0:1775764770.957183:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8c000/0x36f9bdc9e4219a89 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x27c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f111a expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00000080:00010000:3.0:1775764770.957190:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e4219a89 from it ffffafbac1c5baf8 00010000:00010000:3.0:1775764770.957193:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8c000/0x36f9bdc9e4219a89 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x27c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f111a expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:3.0:1775764770.957199:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8c000/0x36f9bdc9e4219a89 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x27c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f111a expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 7 lru_type: 0 00000080:00200008:3.0:1775764770.957313:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file BASEMACH.DOC:[0x200000401:0x27c:0x0](ffff8f9952db5b88), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764770.957317:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x27c:0x0](ffff8f9952db5b88),name=BASEMACH.DOC 00000002:00010000:3.0:1775764770.957331:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x27c:0x0]) in obj [0x200000401:0x278:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764770.957338:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8c000/0x36f9bdc9e4219a89 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x27c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f111a expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 7 lru_type: 0 00010000:00010000:3.0:1775764770.957345:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8c000/0x36f9bdc9e4219a89 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x27c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f111a expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 7 lru_type: 0 00000080:00200000:3.0:1775764770.957350:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x278:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764770.957354:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x27c:0x0](ffff8f9952db5b88) 00000080:00010000:3.0:1775764770.957356:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x27c:0x0](ffff8f9952db5b88) for lock 0x36f9bdc9e4219a89 00000080:00010000:3.0:1775764770.957359:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e4219a89 from it ffffafbac1c5bcb8 00010000:00010000:3.0:1775764770.957362:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8c000/0x36f9bdc9e4219a89 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x27c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f111a expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 7 lru_type: 0 00010000:00010000:3.0:1775764770.957366:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8c000/0x36f9bdc9e4219a89 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x27c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f111a expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 8 lru_type: 0 00000080:00200000:3.0:1775764770.957395:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x27c:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764770.957401:0:19724:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x27c:0x0] 00000008:00010000:3.0:1775764770.957416:0:19724:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1775764770.957419:0:19724:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d5cb0, osc lock ffff8f9966ace4c0, flags 0x400001002 00010000:00010000:3.0:1775764770.957432:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000bb0f9183/0x36f9bdc9e41ca21f lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x3a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ecffc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 1 00010000:00010000:3.0:1775764770.957439:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000bb0f9183/0x36f9bdc9e41ca21f lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x3a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ecffc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 1 00000020:00200000:3.0:1775764770.957446:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1775764770.957449:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000bb0f9183/0x36f9bdc9e41ca21f lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x3a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ecffc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 1 00010000:00010000:3.0:1775764770.957453:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000bb0f9183/0x36f9bdc9e41ca21f lrc: 4/0,0 mode: PW/PW res: [0x280000403:0x3a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ecffc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 21 lru_type: 1 00000020:00200000:3.0:1775764770.957459:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1775764770.957465:0:19724:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x27c:0x0] updating i_size 260096 i_blocks 1 00000020:00010000:3.0:1775764770.957467:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef50000 (ffff8f9970314b48/0) at cl_lock_release():245 00000020:00010000:3.0:1775764770.957469:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef50000 (ffff8f9970314b48/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.957471:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d5cb0 (ffff8f9970314e18/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.957474:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef50000 (ffff8f9970314b48/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764770.957475:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d5cb0 (ffff8f9970314b48/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764770.957486:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x27c:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764770.957499:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.957502:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.957504:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:3.0:1775764770.957506:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.957508:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=16 00000080:00200000:3.0:1775764770.957509:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.957511:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:3.0:1775764770.957512:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.957513:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=WORD, flags=16 00000080:00200000:3.0:1775764770.957514:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x278:0x0](ffff8f9952dbf448), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.957518:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=BASEMACH.DOC, flags=0 00000080:00200000:3.0:1775764770.957520:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x278:0x0](ffff8f9952dbf448), inode mode 41c0 mask 3 00000080:00200000:3.0:1775764770.957524:0:19724:0:(namei.c:2395:ll_unlink()) VFS Op:name=BASEMACH.DOC, dir=[0x200000401:0x278:0x0](ffff8f9952dbf448) 00010000:00010000:3.0:1775764770.957556:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8c000/0x36f9bdc9e4219a89 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x27c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f111a expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 8 lru_type: 0 00000080:00200000:3.0:1775764770.957578:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x27c:0x0]: losing layout lock 00010000:00010000:3.0:1775764770.958035:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8c000/0x36f9bdc9e4219a89 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x27c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f111a expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 8 lru_type: 0 00010000:00010000:3.0:1775764770.958044:0:19724:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764770.958046:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8c000/0x36f9bdc9e4219a89 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x27c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f111a expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 8 lru_type: 0 00000020:00100000:3.0:1775764770.958072:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1775764770.958082:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970ede680 x1862022966931072/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764770.958104:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966931072:192.168.202.146@tcp:36:dbench.0 00000100:00100000:3.0:1775764770.958221:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99707ef180 going to sleep for 11 seconds 00010000:00010000:0.0:1775764770.963014:0:2380:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c40b lrc: 4/1,0 mode: --/CR res: [0x200000401:0x278:0x0].0x0 bits 0x3/0x0 rrc: 6 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f972b expref: -99 pid: 19873 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.963038:0:2380:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c40b lrc: 4/1,0 mode: --/PR res: [0x200000401:0xeb1:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f972b expref: -99 pid: 19873 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.963047:0:2380:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c40b lrc: 4/1,0 mode: --/PR res: [0x200000401:0xeb1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f972b expref: -99 pid: 19873 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.963059:0:2380:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c40b lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xeb1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f972b expref: -99 pid: 19873 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.963071:0:2380:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c40b lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xeb1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f972b expref: -99 pid: 19873 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.963082:0:2380:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c40b lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xeb1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f972b expref: -99 pid: 19873 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.963090:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c40b lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xeb1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f972b expref: -99 pid: 19873 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.963099:0:2380:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c40b lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xeb1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f972b expref: -99 pid: 19873 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764770.963114:0:2380:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994f0a9880 x1862022966930688/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 1472/2440 e 0 to 0 dl 1775764786 ref 1 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1775764770.963126:0:2380:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c40b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xeb1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f972b expref: -99 pid: 19873 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1775764770.963139:0:2380:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c40b from it ffff8f9966bf7d88 00010000:00010000:0.0:1775764770.963143:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c40b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xeb1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f972b expref: -99 pid: 19873 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.963152:0:2380:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c40b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xeb1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f972b expref: -99 pid: 19873 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.963179:0:2380:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c412 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x278:0x0].0x0 bits 0x3/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9747 expref: -99 pid: 19873 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.963189:0:2380:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c412 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x27a:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9747 expref: -99 pid: 19873 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.963196:0:2380:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c412 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x27a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9747 expref: -99 pid: 19873 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.963203:0:2380:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c412 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x27a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9747 expref: -99 pid: 19873 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.963216:0:2380:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c412 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x27a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9747 expref: -99 pid: 19873 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.963224:0:2380:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c412 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x27a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9747 expref: -99 pid: 19873 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.963231:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c412 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x27a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9747 expref: -99 pid: 19873 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.963237:0:2380:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c412 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x27a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9747 expref: -99 pid: 19873 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764770.963247:0:2380:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994f0a9880 x1862022966930688/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 1472/2440 e 0 to 0 dl 1775764786 ref 2 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1775764770.963255:0:2380:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c412 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x27a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9747 expref: -99 pid: 19873 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1775764770.963264:0:2380:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c412 from it ffff8f9966bf5588 00010000:00010000:0.0:1775764770.963268:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c412 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x27a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9747 expref: -99 pid: 19873 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.963280:0:2380:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c412 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x27a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9747 expref: -99 pid: 19873 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.963292:0:2380:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd600/0x36f9bdc9e421c419 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x278:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9763 expref: -99 pid: 19873 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.963306:0:2380:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd600/0x36f9bdc9e421c419 lrc: 4/1,0 mode: --/PR res: [0x200000401:0xeaa:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9763 expref: -99 pid: 19873 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.963313:0:2380:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd600/0x36f9bdc9e421c419 lrc: 4/1,0 mode: --/PR res: [0x200000401:0xeaa:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9763 expref: -99 pid: 19873 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.963319:0:2380:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd600/0x36f9bdc9e421c419 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xeaa:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9763 expref: -99 pid: 19873 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.963327:0:2380:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd600/0x36f9bdc9e421c419 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xeaa:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9763 expref: -99 pid: 19873 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.963335:0:2380:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd600/0x36f9bdc9e421c419 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xeaa:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9763 expref: -99 pid: 19873 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.963342:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd600/0x36f9bdc9e421c419 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xeaa:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9763 expref: -99 pid: 19873 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.963348:0:2380:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd600/0x36f9bdc9e421c419 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xeaa:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9763 expref: -99 pid: 19873 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764770.963357:0:2380:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994f0a9880 x1862022966930688/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 1472/2440 e 0 to 0 dl 1775764786 ref 3 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1775764770.963366:0:2380:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd600/0x36f9bdc9e421c419 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xeaa:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9763 expref: -99 pid: 19873 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1775764770.963381:0:2380:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c419 from it ffff8f9966bf6d88 00010000:00010000:0.0:1775764770.963384:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd600/0x36f9bdc9e421c419 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xeaa:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9763 expref: -99 pid: 19873 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.963391:0:2380:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd600/0x36f9bdc9e421c419 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xeaa:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9763 expref: -99 pid: 19873 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1775764770.963406:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0a9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966930688:192.168.202.146@tcp:63:ll_sa.0 00010000:00010000:0.0:1775764770.963441:0:131:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c40b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xeb1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f972b expref: -99 pid: 19873 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764770.963526:0:131:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0xeb1:0x0](ffff8f9952eeaa08): rc = 0 00000080:00200000:0.0:1775764770.963546:0:131:0:(file.c:6766:ll_layout_conf()) [0x200000401:0xeb1:0x0]: layout version change: 3 -> 3 00000080:00010000:0.0:1775764770.963555:0:131:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xeb1:0x0](ffff8f9952eeaa08) for lock 0x36f9bdc9e421c40b 00000080:00010000:0.0:1775764770.963568:0:131:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c40b from it ffff8f9966bf7d88 00010000:00010000:0.0:1775764770.963571:0:131:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c40b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xeb1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f972b expref: -99 pid: 19873 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.963579:0:131:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c40b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xeb1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f972b expref: -99 pid: 19873 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764770.963595:0:131:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c412 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x27a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9747 expref: -99 pid: 19873 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764770.963639:0:131:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x27a:0x0](ffff8f9952dbba88): rc = 0 00000080:00200000:0.0:1775764770.963655:0:131:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x27a:0x0]: layout version change: 3 -> 3 00000080:00010000:0.0:1775764770.963663:0:131:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x27a:0x0](ffff8f9952dbba88) for lock 0x36f9bdc9e421c412 00000080:00010000:0.0:1775764770.963668:0:131:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c412 from it ffff8f9966bf5588 00010000:00010000:0.0:1775764770.963670:0:131:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c412 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x27a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9747 expref: -99 pid: 19873 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.963677:0:131:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c412 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x27a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9747 expref: -99 pid: 19873 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764770.963692:0:131:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd600/0x36f9bdc9e421c419 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xeaa:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9763 expref: -99 pid: 19873 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764770.963725:0:131:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0xeaa:0x0](ffff8f9952eb2a08): rc = 0 00000080:00200000:1.0:1775764770.964031:0:19874:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xeb1:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764770.964042:0:19874:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xeb1:0x0] 00000008:00010000:1.0:1775764770.964068:0:19874:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:1.0:1775764770.964072:0:19874:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f1c7eb0, osc lock ffff8f995875db50, flags 0x400041012 00010000:00010000:1.0:1775764770.964090:0:19874:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003d3d2f4a/0x36f9bdc9e4219e56 lrc: 3/0,1 mode: PW/PW res: [0x240000403:0x64d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f1d29 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 30 lru_type: 1 00010000:00010000:1.0:1775764770.964104:0:19874:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003d3d2f4a/0x36f9bdc9e4219e56 lrc: 3/0,1 mode: PW/PW res: [0x240000403:0x64d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f1d29 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 30 lru_type: 1 00010000:00010000:1.0:1775764770.964113:0:19874:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003d3d2f4a/0x36f9bdc9e4219e56 lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x64d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5f1d29 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 30 lru_type: 1 00010000:00010000:1.0:1775764770.964120:0:19874:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003d3d2f4a/0x36f9bdc9e4219e56 lrc: 4/0,0 mode: PW/PW res: [0x240000403:0x64d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f1d29 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 31 lru_type: 1 00000020:00010000:1.0:1775764770.964131:0:19874:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_release():245 00000020:00010000:1.0:1775764770.964134:0:19874:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764770.964137:0:19874:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f1c7eb0 (ffff8f99705b8ea8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764770.964140:0:19874:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764770.964142:0:19874:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f1c7eb0 (ffff8f99705b8128/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764770.964163:0:19874:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xeb1:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764770.964179:0:19874:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x27a:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764770.964183:0:19874:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x27a:0x0] 00000008:00010000:1.0:1775764770.964193:0:19874:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:1.0:1775764770.964194:0:19874:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f1c7eb0, osc lock ffff8f995875db50, flags 0x400041012 00010000:00010000:1.0:1775764770.964203:0:19874:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000064603db/0x36f9bdc9e41ca1cb lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x39:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ecf15 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 0 00010000:00010000:1.0:1775764770.964211:0:19874:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000064603db/0x36f9bdc9e41ca1cb lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x39:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ecf15 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 0 00010000:00010000:1.0:1775764770.964218:0:19874:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000064603db/0x36f9bdc9e41ca1cb lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x39:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ecf15 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 0 00010000:00010000:1.0:1775764770.964224:0:19874:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000064603db/0x36f9bdc9e41ca1cb lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x39:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ecf15 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 0 00000020:00010000:1.0:1775764770.964232:0:19874:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_release():245 00000020:00010000:1.0:1775764770.964234:0:19874:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764770.964236:0:19874:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f1c7eb0 (ffff8f99705b8ea8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764770.964239:0:19874:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764770.964241:0:19874:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f1c7eb0 (ffff8f99705b8128/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764770.964250:0:19874:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x27a:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764770.964296:0:131:0:(file.c:6766:ll_layout_conf()) [0x200000401:0xeaa:0x0]: layout version change: 3 -> 3 00000080:00010000:0.0:1775764770.964308:0:131:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xeaa:0x0](ffff8f9952eb2a08) for lock 0x36f9bdc9e421c419 00000080:00010000:0.0:1775764770.964352:0:131:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c419 from it ffff8f9966bf6d88 00010000:00010000:0.0:1775764770.964355:0:131:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd600/0x36f9bdc9e421c419 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xeaa:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9763 expref: -99 pid: 19873 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.964362:0:131:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd600/0x36f9bdc9e421c419 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xeaa:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9763 expref: -99 pid: 19873 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764770.964413:0:19874:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xeaa:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764770.964418:0:19874:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xeaa:0x0] 00000008:00010000:1.0:1775764770.964430:0:19874:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:1.0:1775764770.964432:0:19874:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f1c7eb0, osc lock ffff8f995875db50, flags 0x400041012 00010000:00010000:1.0:1775764770.964441:0:19874:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000a6966465/0x36f9bdc9e4219aba lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x649:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 12288->16383) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f11b4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764770.964451:0:19874:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000a6966465/0x36f9bdc9e4219aba lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x649:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 12288->16383) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f11b4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764770.964459:0:19874:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000a6966465/0x36f9bdc9e4219aba lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x649:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 12288->16383) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5f11b4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764770.964466:0:19874:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000a6966465/0x36f9bdc9e4219aba lrc: 4/0,0 mode: PW/PW res: [0x280000403:0x649:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 12288->16383) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f11b4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00000020:00010000:1.0:1775764770.964475:0:19874:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_release():245 00000020:00010000:1.0:1775764770.964477:0:19874:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764770.964480:0:19874:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f1c7eb0 (ffff8f99705b8ea8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764770.964483:0:19874:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764770.964484:0:19874:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f1c7eb0 (ffff8f99705b8128/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764770.964496:0:19874:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xeaa:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:3.0:1775764770.970263:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966931072:192.168.202.146@tcp:36:dbench.0 00000080:00200000:3.0:1775764770.970338:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x27c:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1775764770.970348:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764770.970363:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:3.0:1775764770.970364:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:3.0:1775764770.970399:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x27c:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764770.970700:0:19724:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x27c:0x0](ffff8f9952db5b88) 00000008:00010000:3.0:1775764770.970735:0:19724:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 1 [cma]time 1775764051 1775764051 0 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000bb0f9183/0x36f9bdc9e41ca21f lrc: 1/0,0 mode: PW/PW res: [0x280000403:0x3a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ecffc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 21 lru_type: 1 00000080:00200000:3.0:1775764770.970816:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.970821:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.970826:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:3.0:1775764770.970830:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.970833:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:3.0:1775764770.970835:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.970836:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:3.0:1775764770.970838:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.970841:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=WORD, flags=17 00000080:00200000:3.0:1775764770.970842:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x278:0x0](ffff8f9952dbf448), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.970852:0:19724:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=TIPS.DOC, dir=[0x200000401:0x278:0x0](ffff8f9952dbf448), flags=1 00000080:00200000:3.0:1775764770.970855:0:19724:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=TIPS.DOC, dir=[0x200000401:0x278:0x0](ffff8f9952dbf448), intent=getattr 00010000:00010000:3.0:1775764770.970870:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c40b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xeb1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f972b expref: -99 pid: 19873 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:3.0:1775764770.970880:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c40b from it ffffafbac1c5b900 00010000:00010000:3.0:1775764770.970883:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c40b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xeb1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f972b expref: -99 pid: 19873 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764770.970892:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c40b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xeb1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f972b expref: -99 pid: 19873 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:3.0:1775764770.970920:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file TIPS.DOC:[0x200000401:0xeb1:0x0](ffff8f9952eeaa08), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764770.970925:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xeb1:0x0](ffff8f9952eeaa08),name=TIPS.DOC 00000002:00010000:3.0:1775764770.971098:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xeb1:0x0]) in obj [0x200000401:0x278:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764770.971108:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c40b lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xeb1:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f972b expref: -99 pid: 19873 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764770.971116:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c40b lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xeb1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f972b expref: -99 pid: 19873 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764770.971121:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x278:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764770.971126:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xeb1:0x0](ffff8f9952eeaa08) 00000080:00010000:3.0:1775764770.971128:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xeb1:0x0](ffff8f9952eeaa08) for lock 0x36f9bdc9e421c40b 00000080:00010000:3.0:1775764770.971132:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c40b from it ffffafbac1c5bcb8 00010000:00010000:3.0:1775764770.971134:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c40b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xeb1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f972b expref: -99 pid: 19873 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764770.971138:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c40b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xeb1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f972b expref: -99 pid: 19873 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1775764770.971157:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xeb1:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764770.971162:0:19724:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xeb1:0x0] 00000008:00010000:3.0:1775764770.971175:0:19724:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1775764770.971178:0:19724:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d5cb0, osc lock ffff8f9966ace4c0, flags 0x400001002 00010000:00010000:3.0:1775764770.971190:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003d3d2f4a/0x36f9bdc9e4219e56 lrc: 3/0,1 mode: PW/PW res: [0x240000403:0x64d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f1d29 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 31 lru_type: 1 00010000:00010000:3.0:1775764770.971197:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003d3d2f4a/0x36f9bdc9e4219e56 lrc: 3/0,1 mode: PW/PW res: [0x240000403:0x64d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f1d29 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 31 lru_type: 1 00000020:00200000:3.0:1775764770.971205:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1775764770.971207:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003d3d2f4a/0x36f9bdc9e4219e56 lrc: 6/0,1 mode: PW/PW res: [0x240000403:0x64d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5f1d29 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 31 lru_type: 1 00010000:00010000:3.0:1775764770.971211:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003d3d2f4a/0x36f9bdc9e4219e56 lrc: 5/0,0 mode: PW/PW res: [0x240000403:0x64d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f1d29 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 32 lru_type: 1 00000020:00200000:3.0:1775764770.971217:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1775764770.971223:0:19724:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0xeb1:0x0] updating i_size 14984 i_blocks 1 00000020:00010000:3.0:1775764770.971226:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef50000 (ffff8f9970314b48/0) at cl_lock_release():245 00000020:00010000:3.0:1775764770.971228:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef50000 (ffff8f9970314b48/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.971230:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d5cb0 (ffff8f9970314e18/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.971232:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef50000 (ffff8f9970314b48/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764770.971234:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d5cb0 (ffff8f9970314b48/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764770.971243:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xeb1:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764770.971257:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.971259:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.971261:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:3.0:1775764770.971263:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.971265:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=16 00000080:00200000:3.0:1775764770.971266:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.971268:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:3.0:1775764770.971269:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.971271:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=WORD, flags=16 00000080:00200000:3.0:1775764770.971272:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x278:0x0](ffff8f9952dbf448), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.971275:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=TIPS.DOC, flags=0 00000080:00200000:3.0:1775764770.971277:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x278:0x0](ffff8f9952dbf448), inode mode 41c0 mask 3 00000080:00200000:3.0:1775764770.971280:0:19724:0:(namei.c:2395:ll_unlink()) VFS Op:name=TIPS.DOC, dir=[0x200000401:0x278:0x0](ffff8f9952dbf448) 00010000:00010000:3.0:1775764770.971304:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c40b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xeb1:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f972b expref: -99 pid: 19873 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1775764770.971317:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0xeb1:0x0]: losing layout lock 00010000:00010000:3.0:1775764770.971332:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fc400/0x36f9bdc9e4219e95 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xeb1:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f1ded expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 10 lru_type: 1 00000080:00200000:3.0:1775764770.971340:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0xeb1:0x0]: losing layout lock 00010000:00010000:3.0:1775764770.971364:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c40b lrc: 1/0,0 mode: --/PR res: [0x200000401:0xeb1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f972b expref: -99 pid: 19873 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1775764770.971369:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fc400/0x36f9bdc9e4219e95 lrc: 1/0,0 mode: --/CR res: [0x200000401:0xeb1:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f1ded expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 10 lru_type: 1 00010000:00010000:3.0:1775764770.971373:0:19724:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1775764770.971374:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c40b lrc: 0/0,0 mode: --/PR res: [0x200000401:0xeb1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f972b expref: -99 pid: 19873 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1775764770.971438:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fc400/0x36f9bdc9e4219e95 lrc: 0/0,0 mode: --/CR res: [0x200000401:0xeb1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f1ded expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 10 lru_type: 1 00000020:00100000:3.0:1775764770.971455:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1775764770.971460:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970680700 x1862022966931328/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764770.971477:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970680700 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966931328:192.168.202.146@tcp:36:dbench.0 00000100:00100000:3.0:1775764770.971540:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99707ef180 going to sleep for 11 seconds 00000100:00100000:3.0:1775764770.976474:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970680700 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966931328:192.168.202.146@tcp:36:dbench.0 00000080:00200000:3.0:1775764770.976548:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xeb1:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1775764770.976561:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764770.976580:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:3.0:1775764770.976582:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000008:00010000:3.0:1775764770.976598:0:19724:0:(osc_cache.c:1167:osc_extent_make_ready()) ### extent: 0000000041d1b24c ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003d3d2f4a/0x36f9bdc9e4219e56 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x64d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f1d29 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 32 lru_type: 1 00000008:00010000:3.0:1775764770.976937:0:19724:0:(osc_cache.c:901:osc_extent_finish()) ### extent: 0000000041d1b24c ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003d3d2f4a/0x36f9bdc9e4219e56 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x64d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f1d29 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 32 lru_type: 1 00000008:00010000:3.0:1775764770.976973:0:19724:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 0000000041d1b24c ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003d3d2f4a/0x36f9bdc9e4219e56 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x64d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f1d29 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 32 lru_type: 1 00000080:00200000:3.0:1775764770.977012:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xeb1:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764770.977084:0:19724:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0xeb1:0x0](ffff8f9952eeaa08) 00000008:00010000:3.0:1775764770.977122:0:19724:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 14984 blocks 1 [cma]time 1775764752 1775764752 1775764712 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003d3d2f4a/0x36f9bdc9e4219e56 lrc: 1/0,0 mode: PW/PW res: [0x240000403:0x64d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f1d29 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 32 lru_type: 1 00000080:00200000:3.0:1775764770.977191:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.977196:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.977201:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:3.0:1775764770.977205:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764770.977210:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:3.0:1775764770.977212:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.977215:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:3.0:1775764770.977217:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.977221:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=WORD, flags=17 00000080:00200000:3.0:1775764770.977223:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x278:0x0](ffff8f9952dbf448), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764770.977232:0:19724:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=CHAP10.DOC, dir=[0x200000401:0x278:0x0](ffff8f9952dbf448), flags=1 00000080:00200000:3.0:1775764770.977235:0:19724:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=CHAP10.DOC, dir=[0x200000401:0x278:0x0](ffff8f9952dbf448), intent=getattr 00010000:00010000:3.0:1775764770.977249:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c412 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x27a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9747 expref: -99 pid: 19873 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:3.0:1775764770.977262:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c412 from it ffffafbac1c5b900 00010000:00010000:3.0:1775764770.977265:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c412 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x27a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9747 expref: -99 pid: 19873 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764770.977273:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c412 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x27a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9747 expref: -99 pid: 19873 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:3.0:1775764770.977308:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file CHAP10.DOC:[0x200000401:0x27a:0x0](ffff8f9952dbba88), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764770.977313:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x27a:0x0](ffff8f9952dbba88),name=CHAP10.DOC 00000002:00010000:3.0:1775764770.977671:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x27a:0x0]) in obj [0x200000401:0x278:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764770.977683:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5600/0x36f9bdc9e42198ad lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x27a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f0b40 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764770.977693:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5600/0x36f9bdc9e42198ad lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x27a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f0b40 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764770.977700:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x278:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764770.977706:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x27a:0x0](ffff8f9952dbba88) 00000080:00010000:3.0:1775764770.977709:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x27a:0x0](ffff8f9952dbba88) for lock 0x36f9bdc9e42198ad 00000080:00010000:3.0:1775764770.977714:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e42198ad from it ffffafbac1c5bcb8 00010000:00010000:3.0:1775764770.977716:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5600/0x36f9bdc9e42198ad lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x27a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f0b40 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764770.977723:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5600/0x36f9bdc9e42198ad lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x27a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f0b40 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764770.978120:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x27a:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1775764770.978129:0:19724:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x27a:0x0] 00000008:00010000:0.0:1775764770.978154:0:19724:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1775764770.978158:0:19724:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0df530, osc lock ffff8f996686f430, flags 0x400001002 00010000:00010000:0.0:1775764770.978175:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000064603db/0x36f9bdc9e41ca1cb lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x39:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ecf15 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 0 00010000:00010000:0.0:1775764770.978186:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000064603db/0x36f9bdc9e41ca1cb lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x39:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ecf15 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 0 00000020:00200000:0.0:1775764770.978197:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1775764770.978201:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000064603db/0x36f9bdc9e41ca1cb lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x39:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ecf15 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 0 00010000:00010000:0.0:1775764770.978208:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000064603db/0x36f9bdc9e41ca1cb lrc: 4/0,0 mode: PW/PW res: [0x280000403:0x39:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ecf15 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 0 00000020:00200000:0.0:1775764770.978218:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1775764770.978226:0:19724:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x27a:0x0] updating i_size 260096 i_blocks 1 00000020:00010000:0.0:1775764770.978230:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970b8bd98 (ffff8f9970314b48/0) at cl_lock_release():245 00000020:00010000:0.0:1775764770.978233:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970b8bd98 (ffff8f9970314b48/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.978236:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0df530 (ffff8f9966dc2368/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764770.978240:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970b8bd98 (ffff8f9970314b48/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764770.978242:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0df530 (ffff8f9970314b48/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764770.978264:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x27a:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764770.978292:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.978296:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.978300:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:0.0:1775764770.978304:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764770.978307:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=16 00000080:00200000:0.0:1775764770.978309:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.978312:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:0.0:1775764770.978315:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.978318:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=WORD, flags=16 00000080:00200000:0.0:1775764770.978320:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x278:0x0](ffff8f9952dbf448), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764770.978325:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=CHAP10.DOC, flags=0 00000080:00200000:0.0:1775764770.978330:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x278:0x0](ffff8f9952dbf448), inode mode 41c0 mask 3 00000080:00200000:0.0:1775764770.978335:0:19724:0:(namei.c:2395:ll_unlink()) VFS Op:name=CHAP10.DOC, dir=[0x200000401:0x278:0x0](ffff8f9952dbf448) 00010000:00010000:0.0:1775764770.978372:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c412 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x27a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f9747 expref: -99 pid: 19873 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1775764770.978488:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x27a:0x0]: losing layout lock 00010000:00010000:0.0:1775764770.978509:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5600/0x36f9bdc9e42198ad lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x27a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f0b40 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764770.978527:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x27a:0x0]: losing layout lock 00010000:00010000:0.0:1775764770.978560:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c412 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x27a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f9747 expref: -99 pid: 19873 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764770.978568:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5600/0x36f9bdc9e42198ad lrc: 1/0,0 mode: --/PR res: [0x200000401:0x27a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f0b40 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764770.978574:0:19724:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1775764770.978577:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c412 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x27a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f9747 expref: -99 pid: 19873 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764770.978587:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5600/0x36f9bdc9e42198ad lrc: 0/0,0 mode: --/PR res: [0x200000401:0x27a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f0b40 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000020:00100000:0.0:1775764770.978606:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1775764770.978670:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f0a9880 x1862022966931712/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764770.978692:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0a9880 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966931712:192.168.202.146@tcp:36:dbench.0 00000100:00100000:0.0:1775764770.978978:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f6c0 going to sleep for 11 seconds 00010000:00010000:0.0:1775764770.984270:0:11210:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1775764770.984276:0:11210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98a00/0x36f9bdc9e421c420 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x27d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f97b7 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.984282:0:11210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970a98a00 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:0.0:1775764770.984285:0:11210:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98a00/0x36f9bdc9e421c420 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x27d:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f97b7 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.984291:0:11210:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98a00/0x36f9bdc9e421c420 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x27d:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f97b7 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.984296:0:11210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98a00/0x36f9bdc9e421c420 lrc: 3/0,0 mode: PR/PR res: [0x200000401:0x27d:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f97b7 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.984305:0:11210:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966bb1c00/0x36f9bdc9e421a0b0 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x27d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f2509 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764770.984322:0:11210:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98a00/0x36f9bdc9e421c420 lrc: 2/0,0 mode: --/PR res: [0x200000401:0x27d:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f97b7 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.984326:0:11210:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764770.984340:0:11210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98a00/0x36f9bdc9e421c420 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x27d:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f97b7 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.984344:0:11210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98a00/0x36f9bdc9e421c420 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x27d:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f97b7 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.984350:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764770.984351:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764770.984353:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764770.984361:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0a9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966932096:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764770.990060:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0a9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966932096:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1775764770.990155:0:19875:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971070400/0x36f9bdc9e4219e8e lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x27b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f1de6 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:0.0:1775764770.990167:0:19875:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971070400/0x36f9bdc9e4219e8e lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x27b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f1de6 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00010000:0.0:1775764770.990176:0:19875:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e4219e8e from it ffffafbac1f2fd48 00010000:00010000:0.0:1775764770.990181:0:19875:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971070400/0x36f9bdc9e4219e8e lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x27b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f1de6 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:0.0:1775764770.990189:0:19875:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971070400/0x36f9bdc9e4219e8e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x27b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f1de6 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:0.0:1775764770.990250:0:19875:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c427 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x277:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 19875 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.990257:0:19875:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c427 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x277:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19875 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.990276:0:19875:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c42e lrc: 3/1,0 mode: --/CR res: [0x200000401:0x277:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 19875 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.990281:0:19875:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c42e lrc: 3/1,0 mode: --/CR res: [0x200000401:0x277:0x0].0x0 bits 0x3/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19875 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.990294:0:19875:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c435 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x277:0x0].0x0 bits 0x0/0x0 rrc: 6 type: IBT flags: 0x10000000000000 pid: 19875 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.990303:0:19875:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c435 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x277:0x0].0x0 bits 0x3/0x0 rrc: 6 type: IBT flags: 0x0 pid: 19875 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764770.990362:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966932480:192.168.202.146@tcp:63:ll_sa.0 00000080:00200000:3.0:1775764770.991306:0:19876:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x27b:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764770.991312:0:19876:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x27b:0x0] 00000008:00010000:3.0:1775764770.991331:0:19876:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:3.0:1775764770.991334:0:19876:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d5cb0, osc lock ffff8f9966ace4c0, flags 0x400041012 00010000:00010000:3.0:1775764770.991346:0:19876:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a2c7fa1e/0x36f9bdc9e42134e6 lrc: 2/1,0 mode: PR/PR res: [0x240000403:0x3a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5dc3eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 0 00010000:00010000:3.0:1775764770.991354:0:19876:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a2c7fa1e/0x36f9bdc9e42134e6 lrc: 2/1,0 mode: PR/PR res: [0x240000403:0x3a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5dc3eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 0 00010000:00010000:3.0:1775764770.991360:0:19876:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a2c7fa1e/0x36f9bdc9e42134e6 lrc: 4/1,0 mode: PR/PR res: [0x240000403:0x3a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5dc3eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 0 00010000:00010000:3.0:1775764770.991364:0:19876:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a2c7fa1e/0x36f9bdc9e42134e6 lrc: 3/0,0 mode: PR/PR res: [0x240000403:0x3a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5dc3eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 0 00000020:00010000:3.0:1775764770.991371:0:19876:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_release():245 00000020:00010000:3.0:1775764770.991373:0:19876:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.991375:0:19876:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d5cb0 (ffff8f9970314758/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.991377:0:19876:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764770.991379:0:19876:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d5cb0 (ffff8f9970314e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764770.991392:0:19876:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x27b:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1775764770.996442:0:2379:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c427 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x277:0x0].0x0 bits 0x3/0x0 rrc: 6 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9820 expref: -99 pid: 19875 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.996458:0:2379:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c427 lrc: 4/1,0 mode: --/PR res: [0x200000401:0xeb2:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9820 expref: -99 pid: 19875 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.996465:0:2379:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c427 lrc: 4/1,0 mode: --/PR res: [0x200000401:0xeb2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9820 expref: -99 pid: 19875 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.996476:0:2379:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c427 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xeb2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9820 expref: -99 pid: 19875 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.996487:0:2379:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c427 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xeb2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9820 expref: -99 pid: 19875 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.996496:0:2379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c427 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xeb2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9820 expref: -99 pid: 19875 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.996503:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c427 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xeb2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9820 expref: -99 pid: 19875 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.996509:0:2379:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c427 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xeb2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9820 expref: -99 pid: 19875 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764770.996524:0:2379:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994f0aa680 x1862022966932480/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 1472/2440 e 0 to 0 dl 1775764786 ref 1 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1775764770.996535:0:2379:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c427 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xeb2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9820 expref: -99 pid: 19875 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1775764770.996544:0:2379:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c427 from it ffff8f9966bf4588 00010000:00010000:0.0:1775764770.996546:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c427 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xeb2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9820 expref: -99 pid: 19875 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.996554:0:2379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c427 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xeb2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9820 expref: -99 pid: 19875 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.996576:0:2379:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c42e lrc: 4/1,0 mode: --/CR res: [0x200000401:0x277:0x0].0x0 bits 0x3/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9843 expref: -99 pid: 19875 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.996585:0:2379:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c42e lrc: 4/1,0 mode: --/PR res: [0x200000401:0x279:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9843 expref: -99 pid: 19875 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.996590:0:2379:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c42e lrc: 4/1,0 mode: --/PR res: [0x200000401:0x279:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9843 expref: -99 pid: 19875 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.996597:0:2379:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c42e lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x279:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9843 expref: -99 pid: 19875 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.996603:0:2379:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c42e lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x279:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9843 expref: -99 pid: 19875 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.996859:0:2379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c42e lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x279:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9843 expref: -99 pid: 19875 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.996866:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c42e lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x279:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9843 expref: -99 pid: 19875 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.996872:0:2379:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c42e lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x279:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9843 expref: -99 pid: 19875 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764770.996880:0:2379:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994f0aa680 x1862022966932480/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 1472/2440 e 0 to 0 dl 1775764786 ref 2 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1775764770.996888:0:2379:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c42e lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x279:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9843 expref: -99 pid: 19875 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1775764770.996896:0:2379:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c42e from it ffff8f9966bf6988 00010000:00010000:0.0:1775764770.996899:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c42e lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x279:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9843 expref: -99 pid: 19875 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.996906:0:2379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c42e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x279:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9843 expref: -99 pid: 19875 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.996920:0:2379:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c435 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x277:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f985f expref: -99 pid: 19875 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.996998:0:2379:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c435 lrc: 4/1,0 mode: --/PR res: [0x200000401:0xeb8:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f985f expref: -99 pid: 19875 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.997011:0:2379:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c435 lrc: 4/1,0 mode: --/PR res: [0x200000401:0xeb8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f985f expref: -99 pid: 19875 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.997018:0:2379:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c435 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xeb8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f985f expref: -99 pid: 19875 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.997025:0:2379:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c435 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xeb8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f985f expref: -99 pid: 19875 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.997032:0:2379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c435 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xeb8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f985f expref: -99 pid: 19875 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.997040:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c435 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xeb8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f985f expref: -99 pid: 19875 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.997046:0:2379:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c435 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xeb8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f985f expref: -99 pid: 19875 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764770.997055:0:2379:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994f0aa680 x1862022966932480/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 1472/2440 e 0 to 0 dl 1775764786 ref 3 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1775764770.997065:0:2379:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c435 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xeb8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f985f expref: -99 pid: 19875 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1775764770.997073:0:2379:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c435 from it ffff8f9966bf5d88 00010000:00010000:0.0:1775764770.997077:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c435 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xeb8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f985f expref: -99 pid: 19875 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764770.997088:0:2379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c435 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xeb8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f985f expref: -99 pid: 19875 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1775764770.997100:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0aa680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966932480:192.168.202.146@tcp:63:ll_sa.0 00010000:00010000:0.0:1775764770.997123:0:131:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c427 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xeb2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9820 expref: -99 pid: 19875 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764770.997178:0:131:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0xeb2:0x0](ffff8f9952eb3248): rc = 0 00000080:00200000:0.0:1775764770.997195:0:131:0:(file.c:6766:ll_layout_conf()) [0x200000401:0xeb2:0x0]: layout version change: 3 -> 3 00000080:00010000:0.0:1775764770.997202:0:131:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xeb2:0x0](ffff8f9952eb3248) for lock 0x36f9bdc9e421c427 00000080:00010000:0.0:1775764770.997211:0:131:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c427 from it ffff8f9966bf4588 00010000:00010000:0.0:1775764770.997214:0:131:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c427 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xeb2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9820 expref: -99 pid: 19875 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.997221:0:131:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c427 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xeb2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9820 expref: -99 pid: 19875 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764770.997235:0:131:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c42e lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x279:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9843 expref: -99 pid: 19875 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764770.997263:0:131:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x279:0x0](ffff8f9952dbc2c8): rc = 0 00000080:00200000:0.0:1775764770.997276:0:131:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x279:0x0]: layout version change: 3 -> 3 00000080:00010000:0.0:1775764770.997283:0:131:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x279:0x0](ffff8f9952dbc2c8) for lock 0x36f9bdc9e421c42e 00000080:00010000:0.0:1775764770.997287:0:131:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c42e from it ffff8f9966bf6988 00010000:00010000:0.0:1775764770.997290:0:131:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c42e lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x279:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9843 expref: -99 pid: 19875 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.997298:0:131:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c42e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x279:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9843 expref: -99 pid: 19875 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764770.997311:0:131:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c435 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xeb8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f985f expref: -99 pid: 19875 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764770.997338:0:131:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0xeb8:0x0](ffff8f9952eb5348): rc = 0 00000080:00200000:0.0:1775764770.997351:0:131:0:(file.c:6766:ll_layout_conf()) [0x200000401:0xeb8:0x0]: layout version change: 3 -> 3 00000080:00010000:0.0:1775764770.997358:0:131:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xeb8:0x0](ffff8f9952eb5348) for lock 0x36f9bdc9e421c435 00000080:00010000:0.0:1775764770.997362:0:131:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c435 from it ffff8f9966bf5d88 00010000:00010000:0.0:1775764770.997364:0:131:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c435 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xeb8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f985f expref: -99 pid: 19875 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764770.997371:0:131:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c435 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xeb8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f985f expref: -99 pid: 19875 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764770.997710:0:19876:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xeb2:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764770.997715:0:19876:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xeb2:0x0] 00000008:00010000:3.0:1775764770.997729:0:19876:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:3.0:1775764770.997731:0:19876:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d5cb0, osc lock ffff8f9966ace4c0, flags 0x400041012 00010000:00010000:3.0:1775764770.997756:0:19876:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b7692b0/0x36f9bdc9e4219ee2 lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x64d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 12288->16383) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f1ecd expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 0 00010000:00010000:3.0:1775764770.997763:0:19876:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b7692b0/0x36f9bdc9e4219ee2 lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x64d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 12288->16383) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f1ecd expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 0 00010000:00010000:3.0:1775764770.997769:0:19876:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b7692b0/0x36f9bdc9e4219ee2 lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x64d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 12288->16383) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5f1ecd expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 0 00010000:00010000:3.0:1775764770.997774:0:19876:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b7692b0/0x36f9bdc9e4219ee2 lrc: 4/0,0 mode: PW/PW res: [0x280000403:0x64d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 12288->16383) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f1ecd expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 1 00000020:00010000:3.0:1775764770.997780:0:19876:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_release():245 00000020:00010000:3.0:1775764770.997782:0:19876:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.997784:0:19876:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d5cb0 (ffff8f9970314758/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.997786:0:19876:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764770.997787:0:19876:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d5cb0 (ffff8f9970314e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764770.997799:0:19876:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xeb2:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764770.997812:0:19876:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x279:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764770.997814:0:19876:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x279:0x0] 00000008:00010000:3.0:1775764770.997821:0:19876:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:3.0:1775764770.997822:0:19876:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d5cb0, osc lock ffff8f9966ace4c0, flags 0x400041012 00010000:00010000:3.0:1775764770.997828:0:19876:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000aac2e8d4/0x36f9bdc9e41ca193 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x39:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ece7b expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:3.0:1775764770.997834:0:19876:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000aac2e8d4/0x36f9bdc9e41ca193 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x39:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ece7b expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:3.0:1775764770.997839:0:19876:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000aac2e8d4/0x36f9bdc9e41ca193 lrc: 4/0,1 mode: PW/PW res: [0x240000403:0x39:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ece7b expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:3.0:1775764770.997844:0:19876:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000aac2e8d4/0x36f9bdc9e41ca193 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x39:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ece7b expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00000020:00010000:3.0:1775764770.997848:0:19876:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_release():245 00000020:00010000:3.0:1775764770.997850:0:19876:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.997851:0:19876:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d5cb0 (ffff8f9970314758/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.997852:0:19876:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764770.997853:0:19876:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d5cb0 (ffff8f9970314e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764770.997859:0:19876:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x279:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764770.997868:0:19876:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xeb8:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764770.997870:0:19876:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xeb8:0x0] 00000008:00010000:3.0:1775764770.997876:0:19876:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:3.0:1775764770.997877:0:19876:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d5cb0, osc lock ffff8f9966ace4c0, flags 0x400041012 00010000:00010000:3.0:1775764770.997881:0:19876:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c29fe1cb/0x36f9bdc9e421a2e7 lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x650:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f2bd1 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:3.0:1775764770.997887:0:19876:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c29fe1cb/0x36f9bdc9e421a2e7 lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x650:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f2bd1 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:3.0:1775764770.997892:0:19876:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c29fe1cb/0x36f9bdc9e421a2e7 lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x650:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5f2bd1 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:3.0:1775764770.997896:0:19876:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c29fe1cb/0x36f9bdc9e421a2e7 lrc: 4/0,0 mode: PW/PW res: [0x280000403:0x650:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f2bd1 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00000020:00010000:3.0:1775764770.997901:0:19876:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_release():245 00000020:00010000:3.0:1775764770.997902:0:19876:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.997903:0:19876:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d5cb0 (ffff8f9970314758/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764770.997904:0:19876:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764770.997906:0:19876:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d5cb0 (ffff8f9970314e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764770.997911:0:19876:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xeb8:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:3.0:1775764771.001603:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970f6b800 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966932864:192.168.202.146@tcp:36:dbench.0 00000080:00200000:3.0:1775764771.001696:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x27b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1775764771.001706:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764771.001722:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:3.0:1775764771.001723:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:3.0:1775764771.001778:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x27b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764771.002243:0:19723:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x27b:0x0](ffff8f9952db3a88) 00000008:00010000:0.0:1775764771.002302:0:19723:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 260096 blocks 515 [cma]time: 1775764051 1775764051 0 by oinfo size 260096 blocks 515 [cma]time 1775764051 1775764051 1775764700 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a2c7fa1e/0x36f9bdc9e42134e6 lrc: 1/0,0 mode: PR/PR res: [0x240000403:0x3a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5dc3eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00000080:00200000:0.0:1775764771.002390:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.002396:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.002402:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:0.0:1775764771.002408:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.002411:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:0.0:1775764771.002414:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.002417:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:0.0:1775764771.002419:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.002422:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=WORD, flags=17 00000080:00200000:0.0:1775764771.002425:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x277:0x0](ffff8f9952e64b08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.002437:0:19723:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=HANDOUT.DOC, dir=[0x200000401:0x277:0x0](ffff8f9952e64b08), flags=1 00000080:00200000:0.0:1775764771.002440:0:19723:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=HANDOUT.DOC, dir=[0x200000401:0x277:0x0](ffff8f9952e64b08), intent=getattr 00010000:00010000:0.0:1775764771.002459:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c427 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xeb2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9820 expref: -99 pid: 19875 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:0.0:1775764771.002471:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c427 from it ffffafbac1c7b900 00010000:00010000:0.0:1775764771.002475:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c427 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xeb2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9820 expref: -99 pid: 19875 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764771.002483:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c427 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xeb2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9820 expref: -99 pid: 19875 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:0.0:1775764771.002516:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file HANDOUT.DOC:[0x200000401:0xeb2:0x0](ffff8f9952eb3248), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764771.002521:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xeb2:0x0](ffff8f9952eb3248),name=HANDOUT.DOC 00000002:00010000:0.0:1775764771.002536:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xeb2:0x0]) in obj [0x200000401:0x277:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764771.002544:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c427 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xeb2:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9820 expref: -99 pid: 19875 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764771.002554:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c427 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xeb2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9820 expref: -99 pid: 19875 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1775764771.002560:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x277:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764771.002565:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xeb2:0x0](ffff8f9952eb3248) 00000080:00010000:0.0:1775764771.002568:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xeb2:0x0](ffff8f9952eb3248) for lock 0x36f9bdc9e421c427 00000080:00010000:0.0:1775764771.002573:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c427 from it ffffafbac1c7bcb8 00010000:00010000:0.0:1775764771.002576:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c427 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xeb2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9820 expref: -99 pid: 19875 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764771.002582:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c427 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xeb2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9820 expref: -99 pid: 19875 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:0.0:1775764771.002602:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xeb2:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1775764771.002627:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xeb2:0x0] 00000008:00010000:0.0:1775764771.002652:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1775764771.002656:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0dfc30, osc lock ffff8f996686f430, flags 0x400001002 00010000:00010000:0.0:1775764771.002673:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b7692b0/0x36f9bdc9e4219ee2 lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x64d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 12288->16383) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f1ecd expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 1 00010000:00010000:0.0:1775764771.002683:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b7692b0/0x36f9bdc9e4219ee2 lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x64d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 12288->16383) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f1ecd expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 1 00000020:00200000:0.0:1775764771.002693:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1775764771.002697:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b7692b0/0x36f9bdc9e4219ee2 lrc: 6/0,1 mode: PW/PW res: [0x280000403:0x64d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 12288->16383) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5f1ecd expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 1 00010000:00010000:0.0:1775764771.002704:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b7692b0/0x36f9bdc9e4219ee2 lrc: 5/0,0 mode: PW/PW res: [0x280000403:0x64d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 12288->16383) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f1ecd expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 26 lru_type: 1 00000020:00200000:0.0:1775764771.002713:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1775764771.002722:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0xeb2:0x0] updating i_size 19054 i_blocks 1 00000020:00010000:0.0:1775764771.002725:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_release():245 00000020:00010000:0.0:1775764771.002729:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764771.002732:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0dfc30 (ffff8f9966dc2128/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764771.002735:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764771.002757:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0dfc30 (ffff8f9966dc2368/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764771.002772:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xeb2:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764771.002792:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.002795:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.002799:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:0.0:1775764771.002802:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.002804:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=16 00000080:00200000:0.0:1775764771.002806:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.002809:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:0.0:1775764771.002811:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.002813:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=WORD, flags=16 00000080:00200000:0.0:1775764771.002815:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x277:0x0](ffff8f9952e64b08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.002819:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=HANDOUT.DOC, flags=0 00000080:00200000:0.0:1775764771.002823:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x277:0x0](ffff8f9952e64b08), inode mode 41c0 mask 3 00000080:00200000:0.0:1775764771.002827:0:19723:0:(namei.c:2395:ll_unlink()) VFS Op:name=HANDOUT.DOC, dir=[0x200000401:0x277:0x0](ffff8f9952e64b08) 00010000:00010000:0.0:1775764771.002865:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c427 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xeb2:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f9820 expref: -99 pid: 19875 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:0.0:1775764771.002886:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0xeb2:0x0]: losing layout lock 00010000:00010000:0.0:1775764771.002905:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970924a00/0x36f9bdc9e4219f13 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xeb2:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f1f59 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 11 lru_type: 1 00000080:00200000:0.0:1775764771.002920:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0xeb2:0x0]: losing layout lock 00010000:00010000:0.0:1775764771.002953:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c427 lrc: 1/0,0 mode: --/PR res: [0x200000401:0xeb2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f9820 expref: -99 pid: 19875 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764771.002965:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970924a00/0x36f9bdc9e4219f13 lrc: 1/0,0 mode: --/CR res: [0x200000401:0xeb2:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f1f59 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 11 lru_type: 1 00010000:00010000:0.0:1775764771.002971:0:19723:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1775764771.002973:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c427 lrc: 0/0,0 mode: --/PR res: [0x200000401:0xeb2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f9820 expref: -99 pid: 19875 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764771.002984:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970924a00/0x36f9bdc9e4219f13 lrc: 0/0,0 mode: --/CR res: [0x200000401:0xeb2:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f1f59 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 11 lru_type: 1 00000020:00100000:0.0:1775764771.003005:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1775764771.003015:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f0aa680 x1862022966933120/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764771.003037:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0aa680 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966933120:192.168.202.146@tcp:36:dbench.0 00000100:00100000:0.0:1775764771.003110:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f000 going to sleep for 11 seconds 00000100:00100000:3.0:1775764771.007950:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970f6ad80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966932608:192.168.202.146@tcp:37:dbench.0 00010000:00010000:3.0:1775764771.008569:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c24b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x287:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b54 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1775764771.008584:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c24b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x287:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b54 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:3.0:1775764771.008667:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.008674:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.008681:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:3.0:1775764771.008688:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.008692:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:3.0:1775764771.008696:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.008700:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:3.0:1775764771.008703:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.008706:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PWRPNT, flags=17 00000080:00200000:3.0:1775764771.008708:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x287:0x0](ffff8f9952dba1c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.008721:0:19724:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=ZD16.BMP, dir=[0x200000401:0x287:0x0](ffff8f9952dba1c8), flags=1 00000080:00200000:3.0:1775764771.008725:0:19724:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=ZD16.BMP, dir=[0x200000401:0x287:0x0](ffff8f9952dba1c8), intent=getattr 00000002:00010000:3.0:1775764771.008774:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ZD16.BMP,[0x0:0x0:0x0]) in obj [0x200000401:0x287:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764771.008821:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c43c lrc: 3/1,0 mode: --/CR res: [0x200000401:0x287:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.008830:0:19724:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c43c lrc: 3/1,0 mode: --/CR res: [0x200000401:0x287:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.008839:0:19724:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c43c lrc: 3/1,0 mode: --/CR res: [0x200000401:0x287:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764771.008860:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970682d80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966933248:192.168.202.146@tcp:101:dbench.0 00000100:00100000:3.0:1775764771.008946:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99707ef000 going to sleep for 11 seconds 00000100:00100000:3.0:1775764771.016661:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970682d80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966933248:192.168.202.146@tcp:101:dbench.0 00010000:00010000:3.0:1775764771.016687:0:19724:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c43c lrc: 4/1,0 mode: --/CR res: [0x200000401:0x287:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f98b3 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.016708:0:19724:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c43c lrc: 4/1,0 mode: --/PR res: [0x200000401:0x28a:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f98b3 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.016715:0:19724:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c43c lrc: 4/1,0 mode: --/PR res: [0x200000401:0x28a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f98b3 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.016727:0:19724:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c43c lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x28a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f98b3 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.016768:0:19724:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c43c lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x28a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f98b3 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.016782:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c43c lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x28a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f98b3 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.016790:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c43c lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x28a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f98b3 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.016797:0:19724:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c43c lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x28a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f98b3 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764771.016809:0:19724:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970682d80 x1862022966933248/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/928 e 0 to 0 dl 1775764787 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'dbench.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1775764771.016820:0:19724:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c43c lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x28a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f98b3 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:3.0:1775764771.016837:0:19724:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c43c lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x28a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f98b3 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.016849:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b9e00/0x36f9bdc9e4218d00 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x28a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5ee40a expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.016860:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b9e00/0x36f9bdc9e4218d00 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x28a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5ee40a expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.016867:0:19724:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c43c lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x28a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f98b3 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.016873:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c43c lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x28a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x2aac6a50e5f98b3 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.016880:0:19724:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c43c lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x28a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x2aac6a50e5f98b3 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.016901:0:19724:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000080:00200000:3.0:1775764771.016908:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x287:0x0] suppgids 0 -1: rc 0 00000080:00200000:3.0:1775764771.017005:0:19724:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x28a:0x0](ffff8f9952dbdb88): rc = 0 00000080:00200000:3.0:1775764771.017025:0:19724:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x28a:0x0]: layout version change: 3 -> 3 00000080:00010000:3.0:1775764771.017035:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x28a:0x0](ffff8f9952dbdb88) for lock 0x36f9bdc9e4218d00 00000080:00010000:3.0:1775764771.017045:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x28a:0x0](ffff8f9952dbdb88) 00000080:00010000:3.0:1775764771.017049:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e4218d00 from it ffffafbac1c5bb20 00010000:00010000:3.0:1775764771.017052:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b9e00/0x36f9bdc9e4218d00 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x28a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5ee40a expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.017065:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b9e00/0x36f9bdc9e4218d00 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x28a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5ee40a expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200008:3.0:1775764771.017111:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file ZD16.BMP:[0x200000401:0x28a:0x0](ffff8f9952dbdb88), request_mask 2047, flags 0, foreign 0 00010000:00010000:1.0:1775764771.017735:0:5572:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764771.017769:0:5572:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c43c lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x28a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f98b3 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.017778:0:5572:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99435d4e00 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:1.0:1775764771.017783:0:5572:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c43c lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x28a:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f98b3 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.017796:0:5572:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c43c lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x28a:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f98b3 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.017805:0:5572:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c43c lrc: 3/0,0 mode: PR/PR res: [0x200000401:0x28a:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f98b3 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.017824:0:5572:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b9e00/0x36f9bdc9e4218d00 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x28a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5ee40a expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764771.017860:0:5572:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c43c lrc: 2/0,0 mode: --/PR res: [0x200000401:0x28a:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f98b3 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.017868:0:5572:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764771.017890:0:5572:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c43c lrc: 1/0,0 mode: --/PR res: [0x200000401:0x28a:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f98b3 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.017898:0:5572:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c43c lrc: 0/0,0 mode: --/PR res: [0x200000401:0x28a:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f98b3 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.017913:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764771.017915:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764771.017917:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1775764771.018402:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x28a:0x0](ffff8f9952dbdb88),name=ZD16.BMP 00000002:00010000:3.0:1775764771.018422:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x28a:0x0]) in obj [0x200000401:0x287:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764771.018433:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b9e00/0x36f9bdc9e4218d00 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x28a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5ee40a expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764771.018444:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b9e00/0x36f9bdc9e4218d00 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x28a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5ee40a expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764771.018451:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x287:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764771.018457:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x28a:0x0](ffff8f9952dbdb88) 00000080:00010000:3.0:1775764771.018461:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x28a:0x0](ffff8f9952dbdb88) for lock 0x36f9bdc9e4218d00 00000080:00010000:3.0:1775764771.018465:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e4218d00 from it ffffafbac1c5bcb8 00010000:00010000:3.0:1775764771.018469:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b9e00/0x36f9bdc9e4218d00 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x28a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5ee40a expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764771.018476:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b9e00/0x36f9bdc9e4218d00 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x28a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5ee40a expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1775764771.018516:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x28a:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764771.018526:0:19724:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x28a:0x0] 00000008:00010000:3.0:1775764771.018555:0:19724:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1775764771.018558:0:19724:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d5cb0, osc lock ffff8f9966ace4c0, flags 0x400001002 00010000:00010000:3.0:1775764771.018576:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000002144360d/0x36f9bdc9e41ca441 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x3f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ed5c1 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1775764771.018587:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000002144360d/0x36f9bdc9e41ca441 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x3f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ed5c1 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:3.0:1775764771.018599:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1775764771.018603:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000002144360d/0x36f9bdc9e41ca441 lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x3f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ed5c1 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764771.018724:0:19877:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c443 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x287:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 19877 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.018732:0:19877:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c443 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x287:0x0].0x0 bits 0x3/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19877 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.018804:0:19877:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c44a lrc: 3/1,0 mode: --/CR res: [0x200000401:0x287:0x0].0x0 bits 0x0/0x0 rrc: 6 type: IBT flags: 0x10000000000000 pid: 19877 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.018810:0:19877:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c44a lrc: 3/1,0 mode: --/CR res: [0x200000401:0x287:0x0].0x0 bits 0x3/0x0 rrc: 6 type: IBT flags: 0x0 pid: 19877 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.018825:0:19877:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c451 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x287:0x0].0x0 bits 0x0/0x0 rrc: 7 type: IBT flags: 0x10000000000000 pid: 19877 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.018831:0:19877:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c451 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x287:0x0].0x0 bits 0x3/0x0 rrc: 7 type: IBT flags: 0x0 pid: 19877 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.018846:0:19877:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421c458 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x287:0x0].0x0 bits 0x0/0x0 rrc: 8 type: IBT flags: 0x10000000000000 pid: 19877 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.018852:0:19877:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421c458 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x287:0x0].0x0 bits 0x3/0x0 rrc: 8 type: IBT flags: 0x0 pid: 19877 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.018867:0:19877:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641200/0x36f9bdc9e421c45f lrc: 3/1,0 mode: --/CR res: [0x200000401:0x287:0x0].0x0 bits 0x0/0x0 rrc: 9 type: IBT flags: 0x10000000000000 pid: 19877 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.018873:0:19877:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641200/0x36f9bdc9e421c45f lrc: 3/1,0 mode: --/CR res: [0x200000401:0x287:0x0].0x0 bits 0x3/0x0 rrc: 9 type: IBT flags: 0x0 pid: 19877 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.018886:0:19877:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641c00/0x36f9bdc9e421c466 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x287:0x0].0x0 bits 0x0/0x0 rrc: 10 type: IBT flags: 0x10000000000000 pid: 19877 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.018892:0:19877:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641c00/0x36f9bdc9e421c466 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x287:0x0].0x0 bits 0x3/0x0 rrc: 10 type: IBT flags: 0x0 pid: 19877 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.019242:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000002144360d/0x36f9bdc9e41ca441 lrc: 4/0,0 mode: PW/PW res: [0x280000403:0x3f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ed5c1 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:3.0:1775764771.019254:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1775764771.019264:0:19724:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x28a:0x0] updating i_size 260096 i_blocks 1 00000020:00010000:3.0:1775764771.019268:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_release():245 00000020:00010000:3.0:1775764771.019271:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.019274:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d5cb0 (ffff8f9970314758/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.019278:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764771.019281:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d5cb0 (ffff8f9970314e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764771.019307:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x28a:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.019346:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.019352:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.019358:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:3.0:1775764771.019364:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.019367:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=16 00000080:00200000:3.0:1775764771.019370:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.019374:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:3.0:1775764771.019376:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.019391:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PWRPNT, flags=16 00000080:00200000:3.0:1775764771.019394:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x287:0x0](ffff8f9952dba1c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.019400:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=ZD16.BMP, flags=0 00000080:00200000:3.0:1775764771.019416:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x287:0x0](ffff8f9952dba1c8), inode mode 41c0 mask 3 00000080:00200000:3.0:1775764771.019421:0:19724:0:(namei.c:2395:ll_unlink()) VFS Op:name=ZD16.BMP, dir=[0x200000401:0x287:0x0](ffff8f9952dba1c8) 00010000:00010000:3.0:1775764771.019461:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c24b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x287:0x0].0x0 bits 0x13/0x0 rrc: 11 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f8b54 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:3.0:1775764771.019475:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3b9c00/0x36f9bdc9e41ca3d1 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x287:0x0].0x0 bits 0x5/0x0 rrc: 11 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ed4c5 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 158 lru_type: 1 00010000:00010000:3.0:1775764771.019485:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3b8400/0x36f9bdc9e41ca409 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x287:0x0].0x0 bits 0x11/0x0 rrc: 11 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ed54a expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 82 lru_type: 1 00010000:00010000:3.0:1775764771.019500:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b9e00/0x36f9bdc9e4218d00 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x28a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5ee40a expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1775764771.019519:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x28a:0x0]: losing layout lock 00010000:00010000:3.0:1775764771.019547:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b9e00/0x36f9bdc9e4218d00 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x28a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5ee40a expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1775764771.019555:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c24b lrc: 1/0,0 mode: --/PR res: [0x200000401:0x287:0x0].0x0 bits 0x13/0x0 rrc: 10 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8b54 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:3.0:1775764771.019561:0:19724:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1775764771.019564:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b9e00/0x36f9bdc9e4218d00 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x28a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5ee40a expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1775764771.019584:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bf200/0x36f9bdc9e421c24b lrc: 0/0,0 mode: --/PR res: [0x200000401:0x287:0x0].0x0 bits 0x13/0x0 rrc: 10 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8b54 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000020:00100000:3.0:1775764771.019597:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 2 is allocated opc 36, max 7 00000100:00100000:3.0:1775764771.019669:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970682d80 x1862022966933760/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764771.019691:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970682d80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966933760:192.168.202.146@tcp:36:dbench.0 00000100:00100000:3.0:1775764771.019794:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99707eff00 going to sleep for 11 seconds 00000100:00100000:1.0:1775764771.023339:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764771.023369:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978358656 00000100:00100000:1.0:1775764771.023376:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978358656 00000100:00100000:1.0:1775764771.023433:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24107 00000100:00100000:1.0:1775764771.023441:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970b51c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+5:5795:x1862022978358656:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764771.023451:0:5562:0:(ldlm_lockd.c:2448:ldlm_callback_handler()) callback on lock 0x36f9bdc9e421c43c - lock disappeared 00010000:00010000:1.0:1775764771.023498:0:5562:0:(ldlm_lockd.c:2345:ldlm_callback_errmsg()) @@@ Operate with invalid parameter, NID=12345-192.168.202.146@tcp lock=0x36f9bdc9e421c43c: rc = 0 req@ffff8f9970b51c00 x1862022978358656/t0(0) o104->3f83546c-01a9-4156-8a97-44c446245d3e@192.168.202.146@tcp:252/0 lens 328/224 e 0 to 0 dl 1775764782 ref 1 fl Interpret:/0/0 rc -22/-22 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764771.023509:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970b51c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+5:5795:x1862022978358656:12345-192.168.202.146@tcp:104: Request processed in 70us (180us total) trans 0 rc -22/-22 00000100:00100000:1.0:1775764771.023515:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24107 00000100:00100000:0.0:1775764771.023763:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970b52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966933376:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764771.023820:0:2380:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1775764771.023827:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970b50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966933504:192.168.202.146@tcp:63:ll_sa.0 00000100:00100000:0.0:1775764771.024000:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0aa680 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966933120:192.168.202.146@tcp:36:dbench.0 00000080:00200000:0.0:1775764771.024073:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xeb2:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764771.024085:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1775764771.024105:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:0.0:1775764771.024106:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000008:00010000:0.0:1775764771.024123:0:19723:0:(osc_cache.c:1167:osc_extent_make_ready()) ### extent: 00000000a634f8b6 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b7692b0/0x36f9bdc9e4219ee2 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x64d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 12288->16383) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f1ecd expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 26 lru_type: 1 00000008:00010000:0.0:1775764771.024160:0:19723:0:(osc_cache.c:901:osc_extent_finish()) ### extent: 00000000a634f8b6 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b7692b0/0x36f9bdc9e4219ee2 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x64d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 12288->16383) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f1ecd expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 26 lru_type: 1 00000008:00010000:0.0:1775764771.024184:0:19723:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a634f8b6 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b7692b0/0x36f9bdc9e4219ee2 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x64d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 12288->16383) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f1ecd expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 26 lru_type: 1 00000080:00200000:0.0:1775764771.024218:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xeb2:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764771.024288:0:19723:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0xeb2:0x0](ffff8f9952eb3248) 00000008:00010000:0.0:1775764771.024327:0:19723:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 19054 blocks 1 [cma]time 1775764754 1775764754 1775764732 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b7692b0/0x36f9bdc9e4219ee2 lrc: 1/0,0 mode: PW/PW res: [0x280000403:0x64d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 12288->16383) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f1ecd expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 26 lru_type: 1 00000080:00200000:0.0:1775764771.024405:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.024411:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.024416:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:0.0:1775764771.024422:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.024425:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:0.0:1775764771.024428:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.024431:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:0.0:1775764771.024434:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.024437:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=WORD, flags=17 00000080:00200000:0.0:1775764771.024439:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x277:0x0](ffff8f9952e64b08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.024451:0:19723:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=CHAP10.DOC, dir=[0x200000401:0x277:0x0](ffff8f9952e64b08), flags=1 00000080:00200000:0.0:1775764771.024456:0:19723:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=CHAP10.DOC, dir=[0x200000401:0x277:0x0](ffff8f9952e64b08), intent=getattr 00010000:00010000:0.0:1775764771.024473:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c42e lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x279:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9843 expref: -99 pid: 19875 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:0.0:1775764771.024491:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c42e from it ffffafbac1c7b900 00010000:00010000:0.0:1775764771.024495:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c42e lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x279:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9843 expref: -99 pid: 19875 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764771.024503:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c42e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x279:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9843 expref: -99 pid: 19875 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:0.0:1775764771.024540:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file CHAP10.DOC:[0x200000401:0x279:0x0](ffff8f9952dbc2c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764771.024545:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x279:0x0](ffff8f9952dbc2c8),name=CHAP10.DOC 00000002:00010000:0.0:1775764771.024561:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x279:0x0]) in obj [0x200000401:0x277:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764771.024571:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9c00/0x36f9bdc9e4219c88 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x279:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f1787 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764771.024582:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9c00/0x36f9bdc9e4219c88 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x279:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f1787 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764771.024589:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x277:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764771.024594:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x279:0x0](ffff8f9952dbc2c8) 00000080:00010000:0.0:1775764771.024597:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x279:0x0](ffff8f9952dbc2c8) for lock 0x36f9bdc9e4219c88 00000080:00010000:0.0:1775764771.024602:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e4219c88 from it ffffafbac1c7bcb8 00010000:00010000:0.0:1775764771.024605:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9c00/0x36f9bdc9e4219c88 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x279:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f1787 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764771.024612:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9c00/0x36f9bdc9e4219c88 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x279:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f1787 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764771.024650:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x279:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1775764771.024657:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x279:0x0] 00000008:00010000:0.0:1775764771.024676:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1775764771.024679:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0df1b0, osc lock ffff8f996686f430, flags 0x400001002 00010000:00010000:0.0:1775764771.024694:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000aac2e8d4/0x36f9bdc9e41ca193 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x39:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ece7b expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:0.0:1775764771.024703:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000aac2e8d4/0x36f9bdc9e41ca193 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x39:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ece7b expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00000020:00200000:0.0:1775764771.024713:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1775764771.024716:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000aac2e8d4/0x36f9bdc9e41ca193 lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x39:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ece7b expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:0.0:1775764771.024722:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000aac2e8d4/0x36f9bdc9e41ca193 lrc: 4/0,0 mode: PW/PW res: [0x240000403:0x39:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ece7b expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00000020:00200000:0.0:1775764771.024731:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1775764771.024761:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x279:0x0] updating i_size 260096 i_blocks 1 00000020:00010000:0.0:1775764771.024765:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_release():245 00000020:00010000:0.0:1775764771.024768:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764771.024771:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0df1b0 (ffff8f9966dc2128/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764771.024774:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764771.024776:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0df1b0 (ffff8f9966dc2368/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764771.024790:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x279:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764771.024808:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.024812:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.024814:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:0.0:1775764771.024817:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.024820:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=16 00000080:00200000:0.0:1775764771.024822:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.024824:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:0.0:1775764771.024827:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.024829:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=WORD, flags=16 00000080:00200000:0.0:1775764771.024831:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x277:0x0](ffff8f9952e64b08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.024836:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=CHAP10.DOC, flags=0 00000080:00200000:0.0:1775764771.024840:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x277:0x0](ffff8f9952e64b08), inode mode 41c0 mask 3 00000080:00200000:0.0:1775764771.024844:0:19723:0:(namei.c:2395:ll_unlink()) VFS Op:name=CHAP10.DOC, dir=[0x200000401:0x277:0x0](ffff8f9952e64b08) 00010000:00010000:0.0:1775764771.024879:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c42e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x279:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f9843 expref: -99 pid: 19875 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1775764771.024901:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x279:0x0]: losing layout lock 00010000:00010000:0.0:1775764771.024920:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9c00/0x36f9bdc9e4219c88 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x279:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f1787 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764771.026440:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x279:0x0]: losing layout lock 00010000:00010000:0.0:1775764771.026498:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c42e lrc: 1/0,0 mode: --/PR res: [0x200000401:0x279:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f9843 expref: -99 pid: 19875 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764771.026507:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9c00/0x36f9bdc9e4219c88 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x279:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f1787 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764771.026513:0:19723:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1775764771.026516:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c42e lrc: 0/0,0 mode: --/PR res: [0x200000401:0x279:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f9843 expref: -99 pid: 19875 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764771.026533:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9c00/0x36f9bdc9e4219c88 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x279:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f1787 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000020:00100000:0.0:1775764771.026554:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1775764771.026566:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f0ab480 x1862022966934016/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764771.026590:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0ab480 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966934016:192.168.202.146@tcp:36:dbench.0 00000100:00100000:0.0:1775764771.026688:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f000 going to sleep for 11 seconds 00000100:00100000:3.0:1775764771.031234:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970682d80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966933760:192.168.202.146@tcp:36:dbench.0 00000080:00200000:3.0:1775764771.031414:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x28a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1775764771.031428:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764771.031450:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:3.0:1775764771.031452:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:3.0:1775764771.031486:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x28a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.032709:0:19724:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x28a:0x0](ffff8f9952dbdb88) 00000008:00010000:3.0:1775764771.032785:0:19724:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 1 [cma]time 1775764052 1775764052 0 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000002144360d/0x36f9bdc9e41ca441 lrc: 1/0,0 mode: PW/PW res: [0x280000403:0x3f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ed5c1 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:0.0:1775764771.033238:0:2380:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c443 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x287:0x0].0x0 bits 0x3/0x0 rrc: 9 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f98eb expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.033252:0:2380:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c443 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x28c:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f98eb expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.033260:0:2380:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c443 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x28c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f98eb expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.033273:0:2380:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c443 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x28c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f98eb expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.033284:0:2380:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c443 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x28c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f98eb expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.033295:0:2380:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c443 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x28c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f98eb expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.033310:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c443 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x28c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f98eb expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.033317:0:2380:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c443 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x28c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f98eb expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764771.033333:0:2380:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970b50700 x1862022966933504/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 2672/4648 e 0 to 0 dl 1775764787 ref 1 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1775764771.033343:0:2380:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c443 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x28c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f98eb expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1775764771.033355:0:2380:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c443 from it ffff8f9970e0bd88 00010000:00010000:0.0:1775764771.033358:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c443 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x28c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f98eb expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.033365:0:2380:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c443 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x28c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f98eb expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764771.033392:0:2380:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c44a lrc: 4/1,0 mode: --/CR res: [0x200000401:0x287:0x0].0x0 bits 0x3/0x0 rrc: 8 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9907 expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.033402:0:2380:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c44a lrc: 4/1,0 mode: --/PR res: [0x200000401:0x28e:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9907 expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.033408:0:2380:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c44a lrc: 4/1,0 mode: --/PR res: [0x200000401:0x28e:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9907 expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.033420:0:2380:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c44a lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x28e:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9907 expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.033427:0:2380:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c44a lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x28e:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9907 expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.033436:0:2380:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c44a lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x28e:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9907 expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.033443:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c44a lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x28e:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9907 expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.033451:0:2380:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c44a lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x28e:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9907 expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764771.033460:0:2380:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970b50700 x1862022966933504/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 2672/4648 e 0 to 0 dl 1775764787 ref 2 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1775764771.033468:0:2380:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c44a lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x28e:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9907 expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1775764771.033476:0:2380:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c44a from it ffff8f9970e0b188 00010000:00010000:0.0:1775764771.033479:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c44a lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x28e:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9907 expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.033485:0:2380:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c44a lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x28e:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9907 expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764771.033500:0:2380:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c451 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x287:0x0].0x0 bits 0x3/0x0 rrc: 7 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9923 expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.033508:0:2380:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c451 lrc: 4/1,0 mode: --/PR res: [0x200000401:0xe98:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9923 expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.033514:0:2380:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c451 lrc: 4/1,0 mode: --/PR res: [0x200000401:0xe98:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9923 expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.033521:0:2380:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c451 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xe98:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9923 expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.033528:0:2380:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c451 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xe98:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9923 expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.033535:0:2380:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c451 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xe98:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9923 expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.033541:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c451 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xe98:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9923 expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.033547:0:2380:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c451 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xe98:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9923 expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764771.033555:0:2380:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970b50700 x1862022966933504/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 2672/4648 e 0 to 0 dl 1775764787 ref 3 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1775764771.033568:0:2380:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c451 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xe98:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9923 expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1775764771.033575:0:2380:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c451 from it ffff8f9970e0b588 00010000:00010000:0.0:1775764771.033578:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c451 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xe98:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9923 expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.033584:0:2380:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c451 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xe98:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9923 expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764771.033597:0:2380:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421c458 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x287:0x0].0x0 bits 0x3/0x0 rrc: 6 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f993f expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.033605:0:2380:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421c458 lrc: 4/1,0 mode: --/PR res: [0x200000401:0xea3:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f993f expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.033611:0:2380:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421c458 lrc: 4/1,0 mode: --/PR res: [0x200000401:0xea3:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f993f expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.033628:0:2380:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421c458 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xea3:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f993f expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.033636:0:2380:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421c458 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xea3:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f993f expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.033645:0:2380:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421c458 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xea3:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f993f expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.033655:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421c458 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xea3:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f993f expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.033661:0:2380:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421c458 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xea3:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f993f expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764771.033669:0:2380:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970b50700 x1862022966933504/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 2672/4648 e 0 to 0 dl 1775764787 ref 4 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1775764771.033675:0:2380:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421c458 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xea3:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f993f expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1775764771.033683:0:2380:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c458 from it ffff8f9970e09d88 00010000:00010000:0.0:1775764771.033686:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421c458 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xea3:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f993f expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.033691:0:2380:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421c458 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xea3:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f993f expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764771.033703:0:2380:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641200/0x36f9bdc9e421c45f lrc: 4/1,0 mode: --/CR res: [0x200000401:0x287:0x0].0x0 bits 0x3/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f995b expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.033711:0:2380:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641200/0x36f9bdc9e421c45f lrc: 4/1,0 mode: --/PR res: [0x200000401:0x28f:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f995b expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.033717:0:2380:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641200/0x36f9bdc9e421c45f lrc: 4/1,0 mode: --/PR res: [0x200000401:0x28f:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f995b expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.033729:0:2380:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641200/0x36f9bdc9e421c45f lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x28f:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f995b expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.033735:0:2380:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641200/0x36f9bdc9e421c45f lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x28f:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f995b expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.033761:0:2380:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641200/0x36f9bdc9e421c45f lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x28f:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f995b expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.033768:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641200/0x36f9bdc9e421c45f lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x28f:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f995b expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.033773:0:2380:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641200/0x36f9bdc9e421c45f lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x28f:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f995b expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764771.033781:0:2380:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970b50700 x1862022966933504/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 2672/4648 e 0 to 0 dl 1775764787 ref 5 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1775764771.033787:0:2380:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641200/0x36f9bdc9e421c45f lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x28f:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f995b expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1775764771.033796:0:2380:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c45f from it ffff8f9970e09988 00010000:00010000:0.0:1775764771.033799:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641200/0x36f9bdc9e421c45f lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x28f:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f995b expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.036178:0:2380:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641200/0x36f9bdc9e421c45f lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x28f:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f995b expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764771.036233:0:2380:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641c00/0x36f9bdc9e421c466 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x287:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9977 expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.036243:0:2380:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641c00/0x36f9bdc9e421c466 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x288:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9977 expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.036248:0:2380:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641c00/0x36f9bdc9e421c466 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x288:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9977 expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.036255:0:2380:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641c00/0x36f9bdc9e421c466 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x288:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9977 expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.036262:0:2380:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641c00/0x36f9bdc9e421c466 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x288:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9977 expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.036268:0:2380:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641c00/0x36f9bdc9e421c466 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x288:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9977 expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.036273:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641c00/0x36f9bdc9e421c466 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x288:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9977 expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.036277:0:2380:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641c00/0x36f9bdc9e421c466 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x288:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9977 expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764771.036288:0:2380:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970b50700 x1862022966933504/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 2672/4648 e 0 to 0 dl 1775764787 ref 6 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1775764771.036298:0:2380:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641c00/0x36f9bdc9e421c466 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x288:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9977 expref: -99 pid: 19877 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1775764771.036308:0:2380:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c466 from it ffff8f9970e08988 00010000:00010000:0.0:1775764771.036310:0:2380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641c00/0x36f9bdc9e421c466 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x288:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9977 expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.036315:0:2380:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641c00/0x36f9bdc9e421c466 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x288:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9977 expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1775764771.036327:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970b50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966933504:192.168.202.146@tcp:63:ll_sa.0 00010000:00010000:0.0:1775764771.036360:0:131:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c443 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x28c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f98eb expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764771.036450:0:131:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x28c:0x0](ffff8f9952db1988): rc = 0 00000080:00200000:0.0:1775764771.036472:0:131:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x28c:0x0]: layout version change: 3 -> 3 00000080:00010000:0.0:1775764771.036481:0:131:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x28c:0x0](ffff8f9952db1988) for lock 0x36f9bdc9e421c443 00000080:00010000:0.0:1775764771.036496:0:131:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c443 from it ffff8f9970e0bd88 00010000:00010000:0.0:1775764771.036500:0:131:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c443 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x28c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f98eb expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764771.036507:0:131:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c443 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x28c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f98eb expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764771.036528:0:131:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c44a lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x28e:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9907 expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764771.036562:0:131:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x28e:0x0](ffff8f9952d63248): rc = 0 00000080:00200000:0.0:1775764771.036575:0:131:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x28e:0x0]: layout version change: 3 -> 3 00000080:00010000:0.0:1775764771.036582:0:131:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x28e:0x0](ffff8f9952d63248) for lock 0x36f9bdc9e421c44a 00000080:00010000:0.0:1775764771.036587:0:131:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c44a from it ffff8f9970e0b188 00010000:00010000:0.0:1775764771.036590:0:131:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c44a lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x28e:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9907 expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764771.036597:0:131:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c44a lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x28e:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9907 expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764771.036612:0:131:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c451 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xe98:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9923 expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764771.036659:0:131:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0xe98:0x0](ffff8f9952eb63c8): rc = 0 00000080:00200000:0.0:1775764771.036673:0:131:0:(file.c:6766:ll_layout_conf()) [0x200000401:0xe98:0x0]: layout version change: 3 -> 3 00000080:00010000:0.0:1775764771.036682:0:131:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xe98:0x0](ffff8f9952eb63c8) for lock 0x36f9bdc9e421c451 00000080:00010000:0.0:1775764771.036687:0:131:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c451 from it ffff8f9970e0b588 00010000:00010000:0.0:1775764771.036690:0:131:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c451 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xe98:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9923 expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764771.036697:0:131:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c451 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xe98:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9923 expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764771.036713:0:131:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421c458 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xea3:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f993f expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764771.036765:0:131:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0xea3:0x0](ffff8f9952eb00c8): rc = 0 00000080:00200000:0.0:1775764771.036779:0:131:0:(file.c:6766:ll_layout_conf()) [0x200000401:0xea3:0x0]: layout version change: 3 -> 3 00000080:00010000:0.0:1775764771.036787:0:131:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xea3:0x0](ffff8f9952eb00c8) for lock 0x36f9bdc9e421c458 00000080:00010000:0.0:1775764771.036790:0:131:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c458 from it ffff8f9970e09d88 00010000:00010000:0.0:1775764771.036793:0:131:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421c458 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xea3:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f993f expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764771.036800:0:131:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421c458 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xea3:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f993f expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764771.036814:0:131:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641200/0x36f9bdc9e421c45f lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x28f:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f995b expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764771.036841:0:131:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x28f:0x0](ffff8f9952dc9148): rc = 0 00000080:00200000:0.0:1775764771.036854:0:131:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x28f:0x0]: layout version change: 3 -> 3 00000080:00010000:0.0:1775764771.036862:0:131:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x28f:0x0](ffff8f9952dc9148) for lock 0x36f9bdc9e421c45f 00000080:00010000:0.0:1775764771.036866:0:131:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c45f from it ffff8f9970e09988 00010000:00010000:0.0:1775764771.036869:0:131:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641200/0x36f9bdc9e421c45f lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x28f:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f995b expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764771.036878:0:131:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641200/0x36f9bdc9e421c45f lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x28f:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f995b expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764771.036892:0:131:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641c00/0x36f9bdc9e421c466 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x288:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9977 expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764771.037057:0:131:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x288:0x0](ffff8f9952db4b08): rc = 0 00000080:00200000:0.0:1775764771.037075:0:131:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x288:0x0]: layout version change: 3 -> 3 00000080:00010000:0.0:1775764771.037084:0:131:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x288:0x0](ffff8f9952db4b08) for lock 0x36f9bdc9e421c466 00000080:00010000:0.0:1775764771.037089:0:131:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c466 from it ffff8f9970e08988 00010000:00010000:0.0:1775764771.037092:0:131:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641c00/0x36f9bdc9e421c466 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x288:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9977 expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764771.037100:0:131:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641c00/0x36f9bdc9e421c466 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x288:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9977 expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000100:00100000:0.0:1775764771.037684:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970b52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966933376:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764771.037723:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0ab480 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966934016:192.168.202.146@tcp:36:dbench.0 00000080:00200000:0.0:1775764771.037804:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x279:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764771.037812:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1775764771.037833:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:0.0:1775764771.037835:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:0.0:1775764771.037858:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x279:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764771.038343:0:19723:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x279:0x0](ffff8f9952dbc2c8) 00000008:00010000:0.0:1775764771.038380:0:19723:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 1 [cma]time 1775764051 1775764051 0 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000aac2e8d4/0x36f9bdc9e41ca193 lrc: 1/0,0 mode: PW/PW res: [0x240000403:0x39:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ece7b expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00000080:00200000:0.0:1775764771.038461:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.038468:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.038474:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:0.0:1775764771.038480:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.038483:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:0.0:1775764771.038486:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.038489:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:0.0:1775764771.038492:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.038495:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=WORD, flags=17 00000080:00200000:0.0:1775764771.038497:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x277:0x0](ffff8f9952e64b08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.038510:0:19723:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=TIPS.DOC, dir=[0x200000401:0x277:0x0](ffff8f9952e64b08), flags=1 00000080:00200000:0.0:1775764771.038515:0:19723:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=TIPS.DOC, dir=[0x200000401:0x277:0x0](ffff8f9952e64b08), intent=getattr 00010000:00010000:0.0:1775764771.038530:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c435 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xeb8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f985f expref: -99 pid: 19875 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:0.0:1775764771.038544:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c435 from it ffffafbac1c7b900 00010000:00010000:0.0:1775764771.038548:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c435 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xeb8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f985f expref: -99 pid: 19875 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764771.038556:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c435 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xeb8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f985f expref: -99 pid: 19875 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:0.0:1775764771.038590:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file TIPS.DOC:[0x200000401:0xeb8:0x0](ffff8f9952eb5348), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764771.038596:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xeb8:0x0](ffff8f9952eb5348),name=TIPS.DOC 00000002:00010000:0.0:1775764771.038612:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xeb8:0x0]) in obj [0x200000401:0x277:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764771.038630:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c435 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xeb8:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f985f expref: -99 pid: 19875 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764771.038639:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c435 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xeb8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f985f expref: -99 pid: 19875 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1775764771.038647:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x277:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764771.038653:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xeb8:0x0](ffff8f9952eb5348) 00000080:00010000:0.0:1775764771.038656:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xeb8:0x0](ffff8f9952eb5348) for lock 0x36f9bdc9e421c435 00000080:00010000:0.0:1775764771.038660:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c435 from it ffffafbac1c7bcb8 00010000:00010000:0.0:1775764771.038663:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c435 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xeb8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f985f expref: -99 pid: 19875 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764771.038670:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c435 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xeb8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f985f expref: -99 pid: 19875 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:0.0:1775764771.038692:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xeb8:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1775764771.038698:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xeb8:0x0] 00000008:00010000:0.0:1775764771.038714:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1775764771.038716:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0df830, osc lock ffff8f996686f430, flags 0x400001002 00010000:00010000:0.0:1775764771.038728:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c29fe1cb/0x36f9bdc9e421a2e7 lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x650:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f2bd1 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:0.0:1775764771.038757:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c29fe1cb/0x36f9bdc9e421a2e7 lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x650:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f2bd1 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00000020:00200000:0.0:1775764771.038769:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1775764771.038772:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c29fe1cb/0x36f9bdc9e421a2e7 lrc: 6/0,1 mode: PW/PW res: [0x280000403:0x650:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5f2bd1 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:0.0:1775764771.038779:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c29fe1cb/0x36f9bdc9e421a2e7 lrc: 5/0,0 mode: PW/PW res: [0x280000403:0x650:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f2bd1 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00000020:00200000:0.0:1775764771.038789:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764771.038826:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0xeb8:0x0] updating i_size 14984 i_blocks 1 00000020:00010000:1.0:1775764771.038832:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_release():245 00000020:00010000:1.0:1775764771.038833:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.038836:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0df830 (ffff8f9966dc2128/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.038838:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970b8bd98 (ffff8f9966dc2368/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764771.038840:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0df830 (ffff8f9966dc2368/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764771.038852:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xeb8:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764771.038868:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.038870:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.038872:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:1.0:1775764771.038874:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.038876:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=16 00000080:00200000:1.0:1775764771.038877:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764771.038879:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:1.0:1775764771.038880:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764771.038882:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=WORD, flags=16 00000080:00200000:1.0:1775764771.038884:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x277:0x0](ffff8f9952e64b08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764771.038887:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=TIPS.DOC, flags=0 00000080:00200000:1.0:1775764771.038889:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x277:0x0](ffff8f9952e64b08), inode mode 41c0 mask 3 00000080:00200000:1.0:1775764771.038893:0:19723:0:(namei.c:2395:ll_unlink()) VFS Op:name=TIPS.DOC, dir=[0x200000401:0x277:0x0](ffff8f9952e64b08) 00010000:00010000:1.0:1775764771.038933:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c435 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xeb8:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f985f expref: -99 pid: 19875 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764771.038966:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0xeb8:0x0]: losing layout lock 00010000:00010000:1.0:1775764771.038985:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080000/0x36f9bdc9e421a31f lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xeb8:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f2c79 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 14 lru_type: 1 00000080:00200000:1.0:1775764771.038998:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0xeb8:0x0]: losing layout lock 00010000:00010000:1.0:1775764771.039030:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c435 lrc: 1/0,0 mode: --/PR res: [0x200000401:0xeb8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f985f expref: -99 pid: 19875 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764771.039037:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080000/0x36f9bdc9e421a31f lrc: 1/0,0 mode: --/CR res: [0x200000401:0xeb8:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f2c79 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 14 lru_type: 1 00010000:00010000:1.0:1775764771.039043:0:19723:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1775764771.039046:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c435 lrc: 0/0,0 mode: --/PR res: [0x200000401:0xeb8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f985f expref: -99 pid: 19875 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764771.039056:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080000/0x36f9bdc9e421a31f lrc: 0/0,0 mode: --/CR res: [0x200000401:0xeb8:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f2c79 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 14 lru_type: 1 00000020:00100000:1.0:1775764771.039073:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1775764771.039079:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970b50a80 x1862022966934272/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764771.039095:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970b50a80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966934272:192.168.202.146@tcp:36:dbench.0 00000100:00100000:1.0:1775764771.039157:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f997130de40 going to sleep for 11 seconds 00000100:00100000:1.0:1775764771.045480:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970b50a80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966934272:192.168.202.146@tcp:36:dbench.0 00000080:00200000:1.0:1775764771.045568:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xeb8:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764771.045581:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:1.0:1775764771.045602:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:1.0:1775764771.045604:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000008:00010000:1.0:1775764771.045632:0:19723:0:(osc_cache.c:1167:osc_extent_make_ready()) ### extent: 00000000967c7bab ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c29fe1cb/0x36f9bdc9e421a2e7 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x650:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f2bd1 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00000008:00010000:1.0:1775764771.045664:0:19723:0:(osc_cache.c:901:osc_extent_finish()) ### extent: 00000000967c7bab ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c29fe1cb/0x36f9bdc9e421a2e7 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x650:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f2bd1 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00000008:00010000:1.0:1775764771.045689:0:19723:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000967c7bab ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c29fe1cb/0x36f9bdc9e421a2e7 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x650:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f2bd1 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00000080:00200000:1.0:1775764771.045727:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xeb8:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764771.046071:0:19723:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0xeb8:0x0](ffff8f9952eb5348) 00000008:00010000:1.0:1775764771.046113:0:19723:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 14984 blocks 1 [cma]time 1775764754 1775764754 1775764714 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c29fe1cb/0x36f9bdc9e421a2e7 lrc: 1/0,0 mode: PW/PW res: [0x280000403:0x650:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f2bd1 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00000080:00200000:1.0:1775764771.046177:0:19723:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x277:0x0](ffff8f9952e64b08) pos/size18446744073709551614/7168 32bit_api 0 00000080:00200008:1.0:1775764771.046197:0:19723:0:(file.c:447:ll_file_release()) START file WORD:[0x200000401:0x277:0x0](ffff8f9952e64b08), flags 304000 00010000:00010000:1.0:1775764771.046230:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3b8c00/0x36f9bdc9e41ca169 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x277:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ecdfd expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764771.046266:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.046272:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.046277:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:1.0:1775764771.046282:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.046286:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:1.0:1775764771.046288:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764771.046292:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:1.0:1775764771.046294:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764771.046297:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PWRPNT, flags=1 00010000:00010000:1.0:1775764771.046310:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c2ad lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8cc7 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00010000:1.0:1775764771.046320:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c2ad from it ffffafbac1c7baf8 00010000:00010000:1.0:1775764771.046323:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c2ad lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8cc7 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764771.046331:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c2ad lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8cc7 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200008:1.0:1775764771.046352:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file PWRPNT:[0x200000401:0x285:0x0](ffff8f9952da3a88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764771.046357:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x285:0x0](ffff8f9952da3a88),name=PWRPNT 00000002:00010000:1.0:1775764771.046374:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x285:0x0]) in obj [0x200000401:0x26c:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764771.046382:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c2ad lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8cc7 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764771.046390:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c2ad lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8cc7 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764771.046397:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26c:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764771.046403:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x285:0x0](ffff8f9952da3a88) 00000080:00010000:1.0:1775764771.046407:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x285:0x0](ffff8f9952da3a88) for lock 0x36f9bdc9e421c2ad 00000080:00010000:1.0:1775764771.046411:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c2ad from it ffffafbac1c7bcb8 00010000:00010000:1.0:1775764771.046414:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c2ad lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8cc7 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764771.046421:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c2ad lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8cc7 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764771.046450:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.046454:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.046457:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=19 00000080:00200000:1.0:1775764771.046459:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.046462:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=19 00000080:00200000:1.0:1775764771.046464:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764771.046466:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=19 00000080:00200000:1.0:1775764771.046469:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764771.046472:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PWRPNT, flags=259 00000080:00200000:1.0:1775764771.046475:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x285:0x0](ffff8f9952da3a88), inode mode 41c0 mask 44 00000080:00200008:1.0:1775764771.046482:0:19723:0:(file.c:1057:ll_file_open()) START file PWRPNT:[0x200000401:0x285:0x0](ffff8f9952da3a88), flags 304000 00000080:00200008:1.0:1775764771.046496:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file PWRPNT:[0x200000401:0x285:0x0](ffff8f9952da3a88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764771.046500:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x285:0x0](ffff8f9952da3a88),name=PWRPNT 00000002:00010000:1.0:1775764771.046507:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x285:0x0]) in obj [0x200000401:0x26c:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764771.046514:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c2ad lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8cc7 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764771.046521:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c2ad lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8cc7 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764771.046527:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26c:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764771.046532:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x285:0x0](ffff8f9952da3a88) 00000080:00010000:1.0:1775764771.046534:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x285:0x0](ffff8f9952da3a88) for lock 0x36f9bdc9e421c2ad 00000080:00010000:1.0:1775764771.046538:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c2ad from it ffffafbac1c7bcd8 00010000:00010000:1.0:1775764771.046541:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c2ad lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8cc7 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764771.046548:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c2ad lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8cc7 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764771.046562:0:19723:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x285:0x0](ffff8f9952da3a88) pos/size0/7168 32bit_api 0 00000002:00010000:1.0:1775764771.046573:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x285:0x0]) in obj [0x200000401:0x285:0x0], intent: readdir flags 0 00010000:00010000:1.0:1775764771.046579:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c2ad lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8cc7 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764771.046586:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c2ad lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8cc7 expref: -99 pid: 19857 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000100:00100000:1.0:1775764771.047316:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970b51180 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966934656:192.168.202.146@tcp:37:dbench.0 00000100:00100000:1.0:1775764771.047447:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f997130de40 going to sleep for 11 seconds 00000100:00100000:3.0:1775764771.057168:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970f68e00 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966934912:192.168.202.146@tcp:36:dbench.0 00000080:00200000:3.0:1775764771.057224:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x28e:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1775764771.057233:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764771.057248:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:3.0:1775764771.057249:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:3.0:1775764771.057274:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x28e:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.057528:0:19724:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x28e:0x0](ffff8f9952d63248) 00000008:00010000:3.0:1775764771.057556:0:19724:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 515 [cma]time 1775764744 1775764732 1775764724 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000007719768c/0x36f9bdc9e41ca4b8 lrc: 1/0,0 mode: PW/PW res: [0x280000403:0x41:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ed6ee expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 0 00000080:00200000:3.0:1775764771.057626:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.057631:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.057635:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:3.0:1775764771.057638:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.057641:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:3.0:1775764771.057644:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.057646:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:3.0:1775764771.057649:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.057652:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PWRPNT, flags=17 00000080:00200000:3.0:1775764771.057654:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x287:0x0](ffff8f9952dba1c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.057661:0:19724:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=NEWPCB.PPT, dir=[0x200000401:0x287:0x0](ffff8f9952dba1c8), flags=1 00000080:00200000:3.0:1775764771.057665:0:19724:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=NEWPCB.PPT, dir=[0x200000401:0x287:0x0](ffff8f9952dba1c8), intent=getattr 00010000:00010000:3.0:1775764771.057677:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c451 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xe98:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9923 expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:3.0:1775764771.057686:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c451 from it ffffafbac1c5b900 00010000:00010000:3.0:1775764771.057688:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c451 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xe98:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9923 expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764771.057693:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c451 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xe98:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9923 expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:3.0:1775764771.057917:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file NEWPCB.PPT:[0x200000401:0xe98:0x0](ffff8f9952eb63c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764771.057922:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xe98:0x0](ffff8f9952eb63c8),name=NEWPCB.PPT 00000002:00010000:3.0:1775764771.057933:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xe98:0x0]) in obj [0x200000401:0x287:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764771.057951:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d48600/0x36f9bdc9e4219152 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xe98:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5ef330 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.057961:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d48600/0x36f9bdc9e4219152 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xe98:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5ef330 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764771.057967:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x287:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764771.057972:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xe98:0x0](ffff8f9952eb63c8) 00000080:00010000:3.0:1775764771.057975:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xe98:0x0](ffff8f9952eb63c8) for lock 0x36f9bdc9e4219152 00000080:00010000:3.0:1775764771.057980:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e4219152 from it ffffafbac1c5bcb8 00010000:00010000:3.0:1775764771.057983:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d48600/0x36f9bdc9e4219152 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xe98:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5ef330 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.057989:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d48600/0x36f9bdc9e4219152 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xe98:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5ef330 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764771.058013:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xe98:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764771.058020:0:19724:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xe98:0x0] 00000008:00010000:3.0:1775764771.058038:0:19724:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1775764771.058041:0:19724:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d5cb0, osc lock ffff8f9966ace4c0, flags 0x400001002 00010000:00010000:3.0:1775764771.058055:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000acba8da0/0x36f9bdc9e4219080 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x640:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5ef074 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:3.0:1775764771.058065:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000acba8da0/0x36f9bdc9e4219080 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x640:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5ef074 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00000020:00200000:3.0:1775764771.058080:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1775764771.058083:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000acba8da0/0x36f9bdc9e4219080 lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x640:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5ef074 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:3.0:1775764771.058090:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000acba8da0/0x36f9bdc9e4219080 lrc: 4/0,0 mode: PW/PW res: [0x280000403:0x640:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5ef074 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00000020:00200000:3.0:1775764771.058100:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1775764771.058108:0:19724:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0xe98:0x0] updating i_size 85026 i_blocks 259 00000020:00010000:3.0:1775764771.058111:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_release():245 00000020:00010000:3.0:1775764771.058113:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.058115:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d5cb0 (ffff8f9970314758/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.058118:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764771.058119:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d5cb0 (ffff8f9970314e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764771.058130:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xe98:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.058146:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.058148:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.058150:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:3.0:1775764771.058152:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.058154:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=16 00000080:00200000:3.0:1775764771.058155:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.058156:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:3.0:1775764771.058158:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.058159:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PWRPNT, flags=16 00000080:00200000:3.0:1775764771.058161:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x287:0x0](ffff8f9952dba1c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.058163:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=NEWPCB.PPT, flags=0 00000080:00200000:3.0:1775764771.058167:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x287:0x0](ffff8f9952dba1c8), inode mode 41c0 mask 3 00000080:00200000:3.0:1775764771.058170:0:19724:0:(namei.c:2395:ll_unlink()) VFS Op:name=NEWPCB.PPT, dir=[0x200000401:0x287:0x0](ffff8f9952dba1c8) 00010000:00010000:3.0:1775764771.058197:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c451 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xe98:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f9923 expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764771.058212:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0xe98:0x0]: losing layout lock 00010000:00010000:3.0:1775764771.058225:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d48600/0x36f9bdc9e4219152 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xe98:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5ef330 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764771.058233:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0xe98:0x0]: losing layout lock 00010000:00010000:3.0:1775764771.058239:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8d800/0x36f9bdc9e42190bf lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xe98:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5ef131 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 1 00000080:00200000:3.0:1775764771.058247:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0xe98:0x0]: losing layout lock 00010000:00010000:3.0:1775764771.058266:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c451 lrc: 1/0,0 mode: --/PR res: [0x200000401:0xe98:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f9923 expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764771.058270:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d48600/0x36f9bdc9e4219152 lrc: 1/0,0 mode: --/PR res: [0x200000401:0xe98:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5ef330 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764771.058274:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8d800/0x36f9bdc9e42190bf lrc: 1/0,0 mode: --/CR res: [0x200000401:0xe98:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5ef131 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 1 00010000:00010000:3.0:1775764771.058277:0:19724:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:3.0:1775764771.058279:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c451 lrc: 0/0,0 mode: --/PR res: [0x200000401:0xe98:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f9923 expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764771.058286:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d48600/0x36f9bdc9e4219152 lrc: 0/0,0 mode: --/PR res: [0x200000401:0xe98:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5ef330 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764771.058295:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8d800/0x36f9bdc9e42190bf lrc: 0/0,0 mode: --/CR res: [0x200000401:0xe98:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5ef131 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 1 00000020:00100000:3.0:1775764771.058309:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1775764771.058315:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970683100 x1862022966935296/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 504/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764771.058326:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970683100 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966935296:192.168.202.146@tcp:36:dbench.0 00000100:00100000:3.0:1775764771.058362:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99707eff00 going to sleep for 11 seconds 00010000:00010000:0.0:1775764771.062704:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1775764771.062710:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99000/0x36f9bdc9e421c46d lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x28d:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f9a18 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.062715:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970a99000 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:0.0:1775764771.062718:0:19801:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99000/0x36f9bdc9e421c46d lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x28d:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f9a18 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.062724:0:19801:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99000/0x36f9bdc9e421c46d lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x28d:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f9a18 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.062730:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99000/0x36f9bdc9e421c46d lrc: 3/0,0 mode: PR/PR res: [0x200000401:0x28d:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f9a18 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.062754:0:19801:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e42194cb lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x28d:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5efe97 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764771.062778:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99000/0x36f9bdc9e421c46d lrc: 2/0,0 mode: --/PR res: [0x200000401:0x28d:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9a18 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.062782:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764771.062796:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99000/0x36f9bdc9e421c46d lrc: 1/0,0 mode: --/PR res: [0x200000401:0x28d:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9a18 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.062801:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99000/0x36f9bdc9e421c46d lrc: 0/0,0 mode: --/PR res: [0x200000401:0x28d:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9a18 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.062807:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764771.062808:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764771.062809:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764771.062822:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966935424:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764771.063425:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970683100 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966935296:192.168.202.146@tcp:36:dbench.0 00000080:00200000:3.0:1775764771.063484:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xe98:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1775764771.063497:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764771.063516:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:3.0:1775764771.063518:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:3.0:1775764771.063546:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xe98:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:0.0:1775764771.065122:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966935424:192.168.202.146@tcp:103:ldlm_bl.0 00000080:00200000:3.0:1775764771.067526:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x28d:0x0](ffff8f9952dbcb08),name=TIPS.PPT 00000002:00010000:3.0:1775764771.067555:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x28d:0x0]) in obj [0x200000401:0x285:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764771.067574:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e42194cb lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x28d:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5efe97 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764771.067588:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e42194cb lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x28d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5efe97 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764771.067594:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x285:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764771.067601:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x28d:0x0](ffff8f9952dbcb08) 00000080:00010000:3.0:1775764771.067604:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x28d:0x0](ffff8f9952dbcb08) for lock 0x36f9bdc9e42194cb 00000080:00010000:3.0:1775764771.067626:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e42194cb from it ffffafbac1c7bcb8 00010000:00010000:3.0:1775764771.067630:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e42194cb lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x28d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5efe97 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764771.067638:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e42194cb lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x28d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5efe97 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1775764771.067692:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x28d:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764771.067702:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x28d:0x0] 00000008:00010000:3.0:1775764771.067730:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1775764771.067733:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d5cb0, osc lock ffff8f9966ace4c0, flags 0x400001002 00010000:00010000:3.0:1775764771.067761:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f98c8360/0x36f9bdc9e41ca495 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x41:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ed69a expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 94 lru_type: 1 00010000:00010000:3.0:1775764771.067770:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f98c8360/0x36f9bdc9e41ca495 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x41:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ed69a expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 94 lru_type: 1 00000020:00200000:3.0:1775764771.067778:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1775764771.067781:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f98c8360/0x36f9bdc9e41ca495 lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x41:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ed69a expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 94 lru_type: 1 00010000:00010000:3.0:1775764771.067786:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f98c8360/0x36f9bdc9e41ca495 lrc: 4/0,0 mode: PW/PW res: [0x240000403:0x41:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ed69a expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 95 lru_type: 1 00000020:00200000:3.0:1775764771.067792:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1775764771.067800:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x28d:0x0] updating i_size 260096 i_blocks 515 00000020:00010000:3.0:1775764771.067803:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_release():245 00000020:00010000:3.0:1775764771.067805:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.067807:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d5cb0 (ffff8f9970314758/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.067810:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764771.067812:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d5cb0 (ffff8f9970314e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764771.067832:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x28d:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.067861:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.067865:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.067868:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:3.0:1775764771.067872:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.067874:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=16 00000080:00200000:3.0:1775764771.067876:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.067878:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:3.0:1775764771.067880:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.067882:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PWRPNT, flags=16 00000080:00200000:3.0:1775764771.067883:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x285:0x0](ffff8f9952da3a88), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.067887:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=TIPS.PPT, flags=0 00010000:00010000:0.0:1775764771.069938:0:19879:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc000/0x36f9bdc9e421c474 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x285:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 19879 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.071227:0:19879:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc000/0x36f9bdc9e421c474 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x285:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19879 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.071255:0:19879:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd800/0x36f9bdc9e421c47b lrc: 3/1,0 mode: --/CR res: [0x200000401:0x285:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 19879 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.071259:0:19879:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd800/0x36f9bdc9e421c47b lrc: 3/1,0 mode: --/CR res: [0x200000401:0x285:0x0].0x0 bits 0x3/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19879 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.071268:0:19879:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc600/0x36f9bdc9e421c482 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x285:0x0].0x0 bits 0x0/0x0 rrc: 6 type: IBT flags: 0x10000000000000 pid: 19879 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.071271:0:19879:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc600/0x36f9bdc9e421c482 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x285:0x0].0x0 bits 0x3/0x0 rrc: 6 type: IBT flags: 0x0 pid: 19879 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.071279:0:19879:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c489 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x285:0x0].0x0 bits 0x0/0x0 rrc: 7 type: IBT flags: 0x10000000000000 pid: 19879 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.071283:0:19879:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c489 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x285:0x0].0x0 bits 0x3/0x0 rrc: 7 type: IBT flags: 0x0 pid: 19879 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.071291:0:19879:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c490 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x285:0x0].0x0 bits 0x0/0x0 rrc: 8 type: IBT flags: 0x10000000000000 pid: 19879 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.071294:0:19879:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c490 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x285:0x0].0x0 bits 0x3/0x0 rrc: 8 type: IBT flags: 0x0 pid: 19879 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.071305:0:19879:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c497 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x285:0x0].0x0 bits 0x0/0x0 rrc: 9 type: IBT flags: 0x10000000000000 pid: 19879 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.071308:0:19879:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c497 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x285:0x0].0x0 bits 0x3/0x0 rrc: 9 type: IBT flags: 0x0 pid: 19879 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764771.071348:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966936320:192.168.202.146@tcp:63:ll_sa.0 00010000:00010000:1.0:1775764771.077019:0:2379:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc000/0x36f9bdc9e421c474 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x285:0x0].0x0 bits 0x3/0x0 rrc: 9 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9a73 expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077039:0:2379:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc000/0x36f9bdc9e421c474 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x290:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9a73 expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077044:0:2379:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc000/0x36f9bdc9e421c474 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x290:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9a73 expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077056:0:2379:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc000/0x36f9bdc9e421c474 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x290:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9a73 expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077063:0:2379:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc000/0x36f9bdc9e421c474 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x290:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9a73 expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077070:0:2379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc000/0x36f9bdc9e421c474 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x290:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9a73 expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077075:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc000/0x36f9bdc9e421c474 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x290:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9a73 expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077080:0:2379:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc000/0x36f9bdc9e421c474 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x290:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9a73 expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764771.077089:0:2379:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994f0ab800 x1862022966936320/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 2672/4648 e 0 to 0 dl 1775764787 ref 1 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:1.0:1775764771.077096:0:2379:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc000/0x36f9bdc9e421c474 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x290:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9a73 expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:1.0:1775764771.077105:0:2379:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c474 from it ffff8f9966bf6588 00010000:00010000:1.0:1775764771.077107:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc000/0x36f9bdc9e421c474 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x290:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9a73 expref: -99 pid: 19879 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077113:0:2379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc000/0x36f9bdc9e421c474 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x290:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9a73 expref: -99 pid: 19879 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764771.077136:0:2379:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd800/0x36f9bdc9e421c47b lrc: 4/1,0 mode: --/CR res: [0x200000401:0x285:0x0].0x0 bits 0x3/0x0 rrc: 8 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9a8f expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077142:0:2379:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd800/0x36f9bdc9e421c47b lrc: 4/1,0 mode: --/PR res: [0x200000401:0xea9:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9a8f expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077146:0:2379:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd800/0x36f9bdc9e421c47b lrc: 4/1,0 mode: --/PR res: [0x200000401:0xea9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9a8f expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077150:0:2379:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd800/0x36f9bdc9e421c47b lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xea9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9a8f expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077154:0:2379:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd800/0x36f9bdc9e421c47b lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xea9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9a8f expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077158:0:2379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd800/0x36f9bdc9e421c47b lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xea9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9a8f expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077162:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd800/0x36f9bdc9e421c47b lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xea9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9a8f expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077166:0:2379:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd800/0x36f9bdc9e421c47b lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xea9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9a8f expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764771.077171:0:2379:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994f0ab800 x1862022966936320/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 2672/4648 e 0 to 0 dl 1775764787 ref 2 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:1.0:1775764771.077179:0:2379:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd800/0x36f9bdc9e421c47b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xea9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9a8f expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:1.0:1775764771.077184:0:2379:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c47b from it ffff8f9966bf6d88 00010000:00010000:1.0:1775764771.077185:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd800/0x36f9bdc9e421c47b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xea9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9a8f expref: -99 pid: 19879 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077189:0:2379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd800/0x36f9bdc9e421c47b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xea9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9a8f expref: -99 pid: 19879 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764771.077196:0:2379:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc600/0x36f9bdc9e421c482 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x285:0x0].0x0 bits 0x3/0x0 rrc: 7 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9aab expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077200:0:2379:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc600/0x36f9bdc9e421c482 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x289:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9aab expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077203:0:2379:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc600/0x36f9bdc9e421c482 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x289:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9aab expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077208:0:2379:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc600/0x36f9bdc9e421c482 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x289:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9aab expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077212:0:2379:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc600/0x36f9bdc9e421c482 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x289:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9aab expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077216:0:2379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc600/0x36f9bdc9e421c482 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x289:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9aab expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077223:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc600/0x36f9bdc9e421c482 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x289:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9aab expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077227:0:2379:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc600/0x36f9bdc9e421c482 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x289:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9aab expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764771.077231:0:2379:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994f0ab800 x1862022966936320/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 2672/4648 e 0 to 0 dl 1775764787 ref 3 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:1.0:1775764771.077235:0:2379:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc600/0x36f9bdc9e421c482 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x289:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9aab expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:1.0:1775764771.077239:0:2379:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c482 from it ffff8f9966bf7188 00010000:00010000:1.0:1775764771.077241:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc600/0x36f9bdc9e421c482 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x289:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9aab expref: -99 pid: 19879 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077244:0:2379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc600/0x36f9bdc9e421c482 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x289:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9aab expref: -99 pid: 19879 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764771.077251:0:2379:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c489 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x285:0x0].0x0 bits 0x3/0x0 rrc: 6 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9ac7 expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077257:0:2379:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c489 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x286:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9ac7 expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077260:0:2379:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c489 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x286:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9ac7 expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077268:0:2379:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c489 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x286:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9ac7 expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077273:0:2379:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c489 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x286:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9ac7 expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077277:0:2379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c489 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x286:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9ac7 expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077280:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c489 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x286:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9ac7 expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077284:0:2379:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c489 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x286:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9ac7 expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764771.077288:0:2379:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994f0ab800 x1862022966936320/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 2672/4648 e 0 to 0 dl 1775764787 ref 4 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:1.0:1775764771.077292:0:2379:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c489 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x286:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9ac7 expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:1.0:1775764771.077296:0:2379:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c489 from it ffff8f9966bf5588 00010000:00010000:1.0:1775764771.077298:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c489 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x286:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9ac7 expref: -99 pid: 19879 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077301:0:2379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c489 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x286:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9ac7 expref: -99 pid: 19879 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764771.077311:0:2379:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c490 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x285:0x0].0x0 bits 0x3/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9ae3 expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077318:0:2379:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c490 lrc: 4/1,0 mode: --/PR res: [0x200000401:0xe9a:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9ae3 expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077322:0:2379:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c490 lrc: 4/1,0 mode: --/PR res: [0x200000401:0xe9a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9ae3 expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077326:0:2379:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c490 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xe9a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9ae3 expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077330:0:2379:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c490 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xe9a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9ae3 expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077333:0:2379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c490 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xe9a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9ae3 expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077337:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c490 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xe9a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9ae3 expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077340:0:2379:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c490 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xe9a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9ae3 expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764771.077344:0:2379:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994f0ab800 x1862022966936320/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 2672/4648 e 0 to 0 dl 1775764787 ref 5 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:1.0:1775764771.077352:0:2379:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c490 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xe9a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9ae3 expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:1.0:1775764771.077358:0:2379:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c490 from it ffff8f9966bf7d88 00010000:00010000:1.0:1775764771.077359:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c490 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xe9a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9ae3 expref: -99 pid: 19879 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077363:0:2379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c490 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xe9a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9ae3 expref: -99 pid: 19879 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764771.077370:0:2379:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c497 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x285:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9aff expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077375:0:2379:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c497 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x28b:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9aff expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077378:0:2379:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c497 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x28b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9aff expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077452:0:2379:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c497 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x28b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9aff expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077457:0:2379:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c497 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x28b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9aff expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077462:0:2379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c497 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x28b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9aff expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077469:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c497 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x28b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9aff expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077473:0:2379:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c497 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x28b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9aff expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764771.077478:0:2379:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994f0ab800 x1862022966936320/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 2672/4648 e 0 to 0 dl 1775764787 ref 6 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:1.0:1775764771.077482:0:2379:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c497 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x28b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9aff expref: -99 pid: 19879 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:1.0:1775764771.077487:0:2379:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c497 from it ffff8f9966bf4588 00010000:00010000:1.0:1775764771.077489:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c497 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x28b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9aff expref: -99 pid: 19879 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.077494:0:2379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c497 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x28b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9aff expref: -99 pid: 19879 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000100:00100000:1.0:1775764771.077505:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966936320:192.168.202.146@tcp:63:ll_sa.0 00010000:00010000:1.0:1775764771.077535:0:21:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc000/0x36f9bdc9e421c474 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x290:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9a73 expref: -99 pid: 19879 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764771.078462:0:21:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x290:0x0](ffff8f9952db3248): rc = 0 00000080:00200000:1.0:1775764771.078494:0:21:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x290:0x0]: layout version change: 3 -> 3 00000080:00010000:1.0:1775764771.078505:0:21:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x290:0x0](ffff8f9952db3248) for lock 0x36f9bdc9e421c474 00000080:00010000:1.0:1775764771.078514:0:21:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c474 from it ffff8f9966bf6588 00010000:00010000:1.0:1775764771.078517:0:21:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc000/0x36f9bdc9e421c474 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x290:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9a73 expref: -99 pid: 19879 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764771.078524:0:21:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc000/0x36f9bdc9e421c474 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x290:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9a73 expref: -99 pid: 19879 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764771.078542:0:21:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd800/0x36f9bdc9e421c47b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xea9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9a8f expref: -99 pid: 19879 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764771.078565:0:21:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0xea9:0x0](ffff8f9952eeb248): rc = 0 00000080:00200000:1.0:1775764771.078573:0:21:0:(file.c:6766:ll_layout_conf()) [0x200000401:0xea9:0x0]: layout version change: 3 -> 3 00000080:00010000:1.0:1775764771.078578:0:21:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xea9:0x0](ffff8f9952eeb248) for lock 0x36f9bdc9e421c47b 00000080:00010000:1.0:1775764771.078580:0:21:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c47b from it ffff8f9966bf6d88 00010000:00010000:1.0:1775764771.078582:0:21:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd800/0x36f9bdc9e421c47b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xea9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9a8f expref: -99 pid: 19879 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764771.078586:0:21:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd800/0x36f9bdc9e421c47b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xea9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9a8f expref: -99 pid: 19879 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764771.078595:0:21:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc600/0x36f9bdc9e421c482 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x289:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9aab expref: -99 pid: 19879 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764771.078808:0:21:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x289:0x0](ffff8f9952db9988): rc = 0 00000080:00200000:1.0:1775764771.078822:0:21:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x289:0x0]: layout version change: 3 -> 3 00000080:00010000:1.0:1775764771.078828:0:21:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x289:0x0](ffff8f9952db9988) for lock 0x36f9bdc9e421c482 00000080:00010000:1.0:1775764771.078831:0:21:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c482 from it ffff8f9966bf7188 00010000:00010000:1.0:1775764771.078833:0:21:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc600/0x36f9bdc9e421c482 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x289:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9aab expref: -99 pid: 19879 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764771.078838:0:21:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc600/0x36f9bdc9e421c482 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x289:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9aab expref: -99 pid: 19879 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764771.078850:0:21:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c489 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x286:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9ac7 expref: -99 pid: 19879 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764771.078867:0:21:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x286:0x0](ffff8f9952db9148): rc = 0 00000080:00200000:1.0:1775764771.078874:0:21:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x286:0x0]: layout version change: 3 -> 3 00000080:00010000:1.0:1775764771.078879:0:21:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x286:0x0](ffff8f9952db9148) for lock 0x36f9bdc9e421c489 00000080:00010000:1.0:1775764771.078881:0:21:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c489 from it ffff8f9966bf5588 00010000:00010000:1.0:1775764771.078883:0:21:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c489 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x286:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9ac7 expref: -99 pid: 19879 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764771.078887:0:21:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c489 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x286:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9ac7 expref: -99 pid: 19879 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764771.078895:0:21:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c490 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xe9a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9ae3 expref: -99 pid: 19879 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764771.078911:0:21:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0xe9a:0x0](ffff8f9952eb7448): rc = 0 00000080:00200000:1.0:1775764771.078918:0:21:0:(file.c:6766:ll_layout_conf()) [0x200000401:0xe9a:0x0]: layout version change: 3 -> 3 00000080:00010000:1.0:1775764771.078922:0:21:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xe9a:0x0](ffff8f9952eb7448) for lock 0x36f9bdc9e421c490 00000080:00010000:1.0:1775764771.078925:0:21:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c490 from it ffff8f9966bf7d88 00010000:00010000:1.0:1775764771.078927:0:21:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c490 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xe9a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9ae3 expref: -99 pid: 19879 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764771.078933:0:21:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c490 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xe9a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9ae3 expref: -99 pid: 19879 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764771.078950:0:21:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c497 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x28b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9aff expref: -99 pid: 19879 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764771.078969:0:21:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x28b:0x0](ffff8f9952dbaa08): rc = 0 00000080:00200000:1.0:1775764771.078977:0:21:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x28b:0x0]: layout version change: 3 -> 3 00000080:00010000:1.0:1775764771.078981:0:21:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x28b:0x0](ffff8f9952dbaa08) for lock 0x36f9bdc9e421c497 00000080:00010000:1.0:1775764771.078983:0:21:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c497 from it ffff8f9966bf4588 00010000:00010000:1.0:1775764771.078985:0:21:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c497 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x28b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9aff expref: -99 pid: 19879 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764771.078989:0:21:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c497 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x28b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9aff expref: -99 pid: 19879 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764771.079019:0:19880:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x290:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764771.079027:0:19880:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x290:0x0] 00000008:00010000:1.0:1775764771.079050:0:19880:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:1.0:1775764771.079053:0:19880:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f1c7430, osc lock ffff8f995875db50, flags 0x400041012 00010000:00010000:1.0:1775764771.079071:0:19880:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000cf9baea5/0x36f9bdc9e41ca4f7 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x42:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ed788 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 22 lru_type: 0 00010000:00010000:1.0:1775764771.079081:0:19880:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000cf9baea5/0x36f9bdc9e41ca4f7 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x42:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ed788 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 22 lru_type: 0 00010000:00010000:1.0:1775764771.079087:0:19880:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000cf9baea5/0x36f9bdc9e41ca4f7 lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x42:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ed788 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 22 lru_type: 0 00010000:00010000:1.0:1775764771.079092:0:19880:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000cf9baea5/0x36f9bdc9e41ca4f7 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x42:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ed788 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 0 00000020:00010000:1.0:1775764771.079101:0:19880:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_release():245 00000020:00010000:1.0:1775764771.079103:0:19880:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.079105:0:19880:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f1c7430 (ffff8f99705b8ea8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.079107:0:19880:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764771.079109:0:19880:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f1c7430 (ffff8f99705b8128/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764771.079122:0:19880:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x290:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764771.079132:0:19880:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xea9:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764771.079135:0:19880:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xea9:0x0] 00000008:00010000:1.0:1775764771.079142:0:19880:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:1.0:1775764771.079143:0:19880:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f1c7430, osc lock ffff8f995875db50, flags 0x400041012 00010000:00010000:1.0:1775764771.079150:0:19880:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ba4d6611/0x36f9bdc9e4219a51 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x649:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f106b expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 0 00010000:00010000:1.0:1775764771.079156:0:19880:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ba4d6611/0x36f9bdc9e4219a51 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x649:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f106b expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 0 00010000:00010000:1.0:1775764771.079163:0:19880:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ba4d6611/0x36f9bdc9e4219a51 lrc: 4/0,1 mode: PW/PW res: [0x240000403:0x649:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5f106b expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 0 00010000:00010000:1.0:1775764771.079171:0:19880:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ba4d6611/0x36f9bdc9e4219a51 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x649:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f106b expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 0 00000020:00010000:1.0:1775764771.079187:0:19880:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_release():245 00000020:00010000:1.0:1775764771.079189:0:19880:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.079191:0:19880:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f1c7430 (ffff8f99705b8ea8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.079194:0:19880:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764771.079196:0:19880:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f1c7430 (ffff8f99705b8128/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764771.079208:0:19880:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xea9:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764771.079223:0:19880:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x289:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764771.079226:0:19880:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x289:0x0] 00000008:00010000:1.0:1775764771.079233:0:19880:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:1.0:1775764771.079235:0:19880:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f1c7430, osc lock ffff8f995875db50, flags 0x400041012 00010000:00010000:1.0:1775764771.079241:0:19880:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a6b29ac0/0x36f9bdc9e42125a4 lrc: 2/1,0 mode: PR/PR res: [0x240000403:0x3f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5d925f expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764771.079247:0:19880:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a6b29ac0/0x36f9bdc9e42125a4 lrc: 2/1,0 mode: PR/PR res: [0x240000403:0x3f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5d925f expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764771.079252:0:19880:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a6b29ac0/0x36f9bdc9e42125a4 lrc: 4/1,0 mode: PR/PR res: [0x240000403:0x3f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5d925f expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764771.079257:0:19880:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a6b29ac0/0x36f9bdc9e42125a4 lrc: 3/0,0 mode: PR/PR res: [0x240000403:0x3f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5d925f expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000020:00010000:1.0:1775764771.079262:0:19880:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_release():245 00000020:00010000:1.0:1775764771.079263:0:19880:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.079264:0:19880:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f1c7430 (ffff8f99705b8ea8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.079267:0:19880:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764771.079268:0:19880:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f1c7430 (ffff8f99705b8128/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764771.079275:0:19880:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x289:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764771.079284:0:19880:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x286:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764771.079286:0:19880:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x286:0x0] 00000008:00010000:1.0:1775764771.079292:0:19880:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:1.0:1775764771.079293:0:19880:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f1c7430, osc lock ffff8f995875db50, flags 0x400041012 00010000:00010000:1.0:1775764771.079299:0:19880:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000039e231d7/0x36f9bdc9e41ca3a7 lrc: 3/0,1 mode: PW/PW res: [0x240000403:0x3e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ed432 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 1 00010000:00010000:1.0:1775764771.079304:0:19880:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000039e231d7/0x36f9bdc9e41ca3a7 lrc: 3/0,1 mode: PW/PW res: [0x240000403:0x3e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ed432 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 1 00010000:00010000:1.0:1775764771.079309:0:19880:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000039e231d7/0x36f9bdc9e41ca3a7 lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x3e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ed432 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 1 00010000:00010000:1.0:1775764771.079313:0:19880:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000039e231d7/0x36f9bdc9e41ca3a7 lrc: 4/0,0 mode: PW/PW res: [0x240000403:0x3e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ed432 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 26 lru_type: 1 00000020:00010000:1.0:1775764771.079318:0:19880:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_release():245 00000020:00010000:1.0:1775764771.079320:0:19880:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.079321:0:19880:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f1c7430 (ffff8f99705b8ea8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.079323:0:19880:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764771.079325:0:19880:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f1c7430 (ffff8f99705b8128/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764771.079331:0:19880:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x286:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764771.079339:0:19880:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xe9a:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764771.079341:0:19880:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xe9a:0x0] 00000008:00010000:1.0:1775764771.079347:0:19880:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:1.0:1775764771.079348:0:19880:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f1c7430, osc lock ffff8f995875db50, flags 0x400041012 00010000:00010000:1.0:1775764771.079353:0:19880:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000004c619a3d/0x36f9bdc9e4219286 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x641:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5ef72e expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764771.079358:0:19880:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000004c619a3d/0x36f9bdc9e4219286 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x641:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5ef72e expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764771.079362:0:19880:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000004c619a3d/0x36f9bdc9e4219286 lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x641:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5ef72e expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764771.079366:0:19880:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000004c619a3d/0x36f9bdc9e4219286 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x641:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5ef72e expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00000020:00010000:1.0:1775764771.079371:0:19880:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_release():245 00000020:00010000:1.0:1775764771.079373:0:19880:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.079374:0:19880:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f1c7430 (ffff8f99705b8ea8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.079376:0:19880:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764771.079378:0:19880:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f1c7430 (ffff8f99705b8128/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764771.079394:0:19880:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xe9a:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764771.079404:0:19880:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x28b:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764771.079406:0:19880:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x28b:0x0] 00000008:00010000:1.0:1775764771.079413:0:19880:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:1.0:1775764771.079414:0:19880:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f1c7430, osc lock ffff8f995875db50, flags 0x400041012 00010000:00010000:1.0:1775764771.079418:0:19880:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000025a0613b/0x36f9bdc9e42127f7 lrc: 2/1,0 mode: PR/PR res: [0x240000403:0x40:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5d9a4d expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764771.079424:0:19880:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000025a0613b/0x36f9bdc9e42127f7 lrc: 2/1,0 mode: PR/PR res: [0x240000403:0x40:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5d9a4d expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764771.079429:0:19880:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000025a0613b/0x36f9bdc9e42127f7 lrc: 4/1,0 mode: PR/PR res: [0x240000403:0x40:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5d9a4d expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764771.079434:0:19880:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000025a0613b/0x36f9bdc9e42127f7 lrc: 3/0,0 mode: PR/PR res: [0x240000403:0x40:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5d9a4d expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00000020:00010000:1.0:1775764771.079438:0:19880:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_release():245 00000020:00010000:1.0:1775764771.079440:0:19880:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.079441:0:19880:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f1c7430 (ffff8f99705b8ea8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.079442:0:19880:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764771.079443:0:19880:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f1c7430 (ffff8f99705b8128/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764771.079449:0:19880:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x28b:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:3.0:1775764771.081908:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966936192:192.168.202.146@tcp:36:dbench.0 00000080:00200000:3.0:1775764771.081992:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x28f:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1775764771.082004:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764771.082030:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:3.0:1775764771.082032:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:3.0:1775764771.082070:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x28f:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.082318:0:19724:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x28f:0x0](ffff8f9952dc9148) 00000008:00010000:3.0:1775764771.082352:0:19724:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 260096 blocks 515 [cma]time: 1775764053 1775764053 0 by oinfo size 260096 blocks 515 [cma]time 1775764053 1775764053 1775764696 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000083a42736/0x36f9bdc9e4212bbd lrc: 1/0,0 mode: PR/PR res: [0x240000403:0x42:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5da6be expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00000080:00200000:3.0:1775764771.082421:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.082425:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.082430:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:3.0:1775764771.082435:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.082437:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:3.0:1775764771.082439:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.082441:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:3.0:1775764771.082443:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.082445:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PWRPNT, flags=17 00000080:00200000:3.0:1775764771.082447:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x287:0x0](ffff8f9952dba1c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.082455:0:19724:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=PCBENCHM.PPT, dir=[0x200000401:0x287:0x0](ffff8f9952dba1c8), flags=1 00000080:00200000:3.0:1775764771.082458:0:19724:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=PCBENCHM.PPT, dir=[0x200000401:0x287:0x0](ffff8f9952dba1c8), intent=getattr 00010000:00010000:3.0:1775764771.082472:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641c00/0x36f9bdc9e421c466 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x288:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9977 expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:3.0:1775764771.082481:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c466 from it ffffafbac1c5b900 00010000:00010000:3.0:1775764771.082484:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641c00/0x36f9bdc9e421c466 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x288:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9977 expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764771.082490:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641c00/0x36f9bdc9e421c466 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x288:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9977 expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:3.0:1775764771.082510:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file PCBENCHM.PPT:[0x200000401:0x288:0x0](ffff8f9952db4b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764771.082514:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x288:0x0](ffff8f9952db4b08),name=PCBENCHM.PPT 00000002:00010000:3.0:1775764771.082526:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x288:0x0]) in obj [0x200000401:0x287:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764771.082533:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8ce00/0x36f9bdc9e4218b2b lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x288:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5edafd expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.082541:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8ce00/0x36f9bdc9e4218b2b lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x288:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5edafd expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764771.082549:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x287:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764771.082552:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x288:0x0](ffff8f9952db4b08) 00000080:00010000:3.0:1775764771.082555:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x288:0x0](ffff8f9952db4b08) for lock 0x36f9bdc9e4218b2b 00000080:00010000:3.0:1775764771.082558:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e4218b2b from it ffffafbac1c5bcb8 00010000:00010000:3.0:1775764771.082560:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8ce00/0x36f9bdc9e4218b2b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x288:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5edafd expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.082564:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8ce00/0x36f9bdc9e4218b2b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x288:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5edafd expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764771.082580:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x288:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764771.082585:0:19724:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x288:0x0] 00000008:00010000:3.0:1775764771.082599:0:19724:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1775764771.082602:0:19724:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d5cb0, osc lock ffff8f9966ace4c0, flags 0x400001002 00010000:00010000:3.0:1775764771.083392:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c11d69a0/0x36f9bdc9e41ca3f4 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x3e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ed520 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 28 lru_type: 1 00010000:00010000:3.0:1775764771.083405:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c11d69a0/0x36f9bdc9e41ca3f4 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x3e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ed520 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 28 lru_type: 1 00000020:00200000:3.0:1775764771.083413:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1775764771.083415:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c11d69a0/0x36f9bdc9e41ca3f4 lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x3e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ed520 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 28 lru_type: 1 00010000:00010000:3.0:1775764771.083421:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c11d69a0/0x36f9bdc9e41ca3f4 lrc: 4/0,0 mode: PW/PW res: [0x280000403:0x3e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ed520 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 29 lru_type: 1 00000020:00200000:3.0:1775764771.083432:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1775764771.083441:0:19724:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x288:0x0] updating i_size 260096 i_blocks 515 00000020:00010000:3.0:1775764771.083444:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_release():245 00000020:00010000:3.0:1775764771.083446:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.083448:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d5cb0 (ffff8f9970314758/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.083452:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764771.083453:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d5cb0 (ffff8f9970314e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764771.083469:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x288:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.083490:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.083493:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.083495:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:3.0:1775764771.083499:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.083500:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=16 00000080:00200000:3.0:1775764771.083502:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.083504:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:3.0:1775764771.083505:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.083507:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PWRPNT, flags=16 00000080:00200000:3.0:1775764771.083508:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x287:0x0](ffff8f9952dba1c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.083512:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PCBENCHM.PPT, flags=0 00000080:00200000:3.0:1775764771.083514:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x287:0x0](ffff8f9952dba1c8), inode mode 41c0 mask 3 00000080:00200000:3.0:1775764771.083517:0:19724:0:(namei.c:2395:ll_unlink()) VFS Op:name=PCBENCHM.PPT, dir=[0x200000401:0x287:0x0](ffff8f9952dba1c8) 00010000:00010000:3.0:1775764771.083547:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641c00/0x36f9bdc9e421c466 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x288:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f9977 expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764771.083563:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x288:0x0]: losing layout lock 00010000:00010000:3.0:1775764771.083578:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8ce00/0x36f9bdc9e4218b2b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x288:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5edafd expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764771.083587:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x288:0x0]: losing layout lock 00010000:00010000:3.0:1775764771.083592:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080c00/0x36f9bdc9e41cb287 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x288:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e4efaa4 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764771.083599:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x288:0x0]: losing layout lock 00010000:00010000:3.0:1775764771.083724:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641c00/0x36f9bdc9e421c466 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x288:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f9977 expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764771.083729:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8ce00/0x36f9bdc9e4218b2b lrc: 1/0,0 mode: --/PR res: [0x200000401:0x288:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5edafd expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764771.083733:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080c00/0x36f9bdc9e41cb287 lrc: 1/0,0 mode: --/CR res: [0x200000401:0x288:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e4efaa4 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764771.083835:0:19724:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:3.0:1775764771.083838:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641c00/0x36f9bdc9e421c466 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x288:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f9977 expref: -99 pid: 19877 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764771.083848:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8ce00/0x36f9bdc9e4218b2b lrc: 0/0,0 mode: --/PR res: [0x200000401:0x288:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5edafd expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764771.083853:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f080c00/0x36f9bdc9e41cb287 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x288:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e4efaa4 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000020:00100000:3.0:1775764771.083870:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1775764771.083877:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970683b80 x1862022966936576/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 504/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764771.083902:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970683b80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966936576:192.168.202.146@tcp:36:dbench.0 00000100:00100000:3.0:1775764771.085161:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99707ef000 going to sleep for 11 seconds 00000100:00100000:3.0:1775764771.093914:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970683b80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966936576:192.168.202.146@tcp:36:dbench.0 00000080:00200000:3.0:1775764771.093985:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x288:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1775764771.093996:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764771.094011:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:3.0:1775764771.094012:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:3.0:1775764771.094038:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x288:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.094310:0:19724:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x288:0x0](ffff8f9952db4b08) 00000008:00010000:3.0:1775764771.094346:0:19724:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 515 [cma]time 1775764739 1775764727 1775764719 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c11d69a0/0x36f9bdc9e41ca3f4 lrc: 1/0,0 mode: PW/PW res: [0x280000403:0x3e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ed520 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 29 lru_type: 1 00000080:00200000:3.0:1775764771.094398:0:19724:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x287:0x0](ffff8f9952dba1c8) pos/size18446744073709551614/7168 32bit_api 0 00000080:00200008:3.0:1775764771.094413:0:19724:0:(file.c:447:ll_file_release()) START file PWRPNT:[0x200000401:0x287:0x0](ffff8f9952dba1c8), flags 304000 00010000:00010000:3.0:1775764771.094433:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3b9c00/0x36f9bdc9e41ca3d1 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x287:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ed4c5 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 159 lru_type: 1 00000080:00200000:3.0:1775764771.094464:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.094468:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.094472:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:3.0:1775764771.094476:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.094478:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:3.0:1775764771.094480:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.094482:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:3.0:1775764771.094484:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.094487:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=WORDPRO, flags=1 00010000:00010000:3.0:1775764771.094498:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480400/0x36f9bdc9e421c252 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b70 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00010000:3.0:1775764771.094505:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c252 from it ffffafbac1c5baf8 00010000:00010000:3.0:1775764771.094508:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480400/0x36f9bdc9e421c252 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b70 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764771.094513:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480400/0x36f9bdc9e421c252 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b70 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200008:3.0:1775764771.094531:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file WORDPRO:[0x200000401:0x2a9:0x0](ffff8f9952da6c08), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764771.094535:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2a9:0x0](ffff8f9952da6c08),name=WORDPRO 00000002:00010000:3.0:1775764771.094548:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2a9:0x0]) in obj [0x200000401:0x26b:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764771.094553:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480400/0x36f9bdc9e421c252 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b70 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764771.094558:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480400/0x36f9bdc9e421c252 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b70 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764771.094563:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26b:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764771.094566:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2a9:0x0](ffff8f9952da6c08) 00000080:00010000:3.0:1775764771.094568:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2a9:0x0](ffff8f9952da6c08) for lock 0x36f9bdc9e421c252 00000080:00010000:3.0:1775764771.094571:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c252 from it ffffafbac1c5bcb8 00010000:00010000:3.0:1775764771.094573:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480400/0x36f9bdc9e421c252 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b70 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764771.094577:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480400/0x36f9bdc9e421c252 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b70 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1775764771.094600:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.094603:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.094604:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=19 00000080:00200000:3.0:1775764771.094606:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.094607:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=19 00000080:00200000:3.0:1775764771.094609:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.094611:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=19 00000080:00200000:3.0:1775764771.094613:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.094629:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=WORDPRO, flags=259 00000080:00200000:3.0:1775764771.094633:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2a9:0x0](ffff8f9952da6c08), inode mode 41c0 mask 44 00000080:00200008:3.0:1775764771.094640:0:19724:0:(file.c:1057:ll_file_open()) START file WORDPRO:[0x200000401:0x2a9:0x0](ffff8f9952da6c08), flags 304000 00000080:00200008:3.0:1775764771.094655:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file WORDPRO:[0x200000401:0x2a9:0x0](ffff8f9952da6c08), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764771.094658:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2a9:0x0](ffff8f9952da6c08),name=WORDPRO 00000002:00010000:3.0:1775764771.094665:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2a9:0x0]) in obj [0x200000401:0x26b:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764771.094671:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480400/0x36f9bdc9e421c252 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b70 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1775764771.094678:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480400/0x36f9bdc9e421c252 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b70 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1775764771.094683:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26b:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764771.094686:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2a9:0x0](ffff8f9952da6c08) 00000080:00010000:3.0:1775764771.094688:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2a9:0x0](ffff8f9952da6c08) for lock 0x36f9bdc9e421c252 00000080:00010000:3.0:1775764771.094690:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c252 from it ffffafbac1c5bcd8 00010000:00010000:3.0:1775764771.094692:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480400/0x36f9bdc9e421c252 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b70 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1775764771.094700:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480400/0x36f9bdc9e421c252 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b70 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1775764771.094710:0:19724:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2a9:0x0](ffff8f9952da6c08) pos/size0/7168 32bit_api 0 00000002:00010000:3.0:1775764771.094717:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2a9:0x0]) in obj [0x200000401:0x2a9:0x0], intent: readdir flags 0 00010000:00010000:3.0:1775764771.094720:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480400/0x36f9bdc9e421c252 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b70 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1775764771.094725:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480400/0x36f9bdc9e421c252 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b70 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000100:00100000:3.0:1775764771.095207:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970681f80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966936960:192.168.202.146@tcp:37:dbench.0 00000100:00100000:3.0:1775764771.095282:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99707ef000 going to sleep for 11 seconds 00000100:00100000:3.0:1775764771.109416:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970681f80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966936960:192.168.202.146@tcp:37:dbench.0 00010000:00010000:3.0:1775764771.110487:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480400/0x36f9bdc9e421c252 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8b70 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1775764771.110502:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480400/0x36f9bdc9e421c252 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8b70 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00000080:00200000:3.0:1775764771.110568:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.110574:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.110581:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:3.0:1775764771.110587:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.110591:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:3.0:1775764771.110594:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.110597:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:3.0:1775764771.110600:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.110602:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=WORDPRO, flags=17 00000080:00200000:3.0:1775764771.110605:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2a9:0x0](ffff8f9952da6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.110608:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=NEWS1_1.LWP, flags=1 00000080:00200008:3.0:1775764771.110627:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file NEWS1_1.LWP:[0x200000401:0x2af:0x0](ffff8f9952d8b248), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764771.111325:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2af:0x0](ffff8f9952d8b248),name=NEWS1_1.LWP 00000002:00010000:3.0:1775764771.111344:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2af:0x0]) in obj [0x200000401:0x2a9:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764771.111359:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971227200/0x36f9bdc9e42168e8 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2af:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e6618 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764771.111365:0:19881:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fce00/0x36f9bdc9e421665d lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2ad:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e5e07 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1775764771.111371:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971227200/0x36f9bdc9e42168e8 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2af:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e6618 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764771.111373:0:19881:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fce00/0x36f9bdc9e421665d lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2ad:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e5e07 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00010000:0.0:1775764771.111381:0:19881:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421665d from it ffffafbac1f2fd48 00010000:00010000:0.0:1775764771.111385:0:19881:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fce00/0x36f9bdc9e421665d lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2ad:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5e5e07 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1775764771.111385:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2a9:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764771.111391:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2af:0x0](ffff8f9952d8b248) 00010000:00010000:0.0:1775764771.111392:0:19881:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fce00/0x36f9bdc9e421665d lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ad:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e5e07 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00010000:3.0:1775764771.111394:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2af:0x0](ffff8f9952d8b248) for lock 0x36f9bdc9e42168e8 00000080:00010000:3.0:1775764771.111400:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e42168e8 from it ffffafbac1c5bcb8 00010000:00010000:3.0:1775764771.111402:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971227200/0x36f9bdc9e42168e8 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2af:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5e6618 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764771.111409:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971227200/0x36f9bdc9e42168e8 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2af:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e6618 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764771.111453:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2af:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764771.111463:0:19724:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2af:0x0] 00000008:00010000:3.0:1775764771.111489:0:19724:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1775764771.111492:0:19724:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d5cb0, osc lock ffff8f9966ace4c0, flags 0x400001002 00010000:00010000:3.0:1775764771.111508:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000009e8ce381/0x36f9bdc9e41ca973 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x50:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee327 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 128 lru_type: 1 00010000:00010000:3.0:1775764771.111521:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000009e8ce381/0x36f9bdc9e41ca973 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x50:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee327 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 128 lru_type: 1 00000020:00200000:3.0:1775764771.111533:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1775764771.111537:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000009e8ce381/0x36f9bdc9e41ca973 lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x50:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ee327 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 128 lru_type: 1 00010000:00010000:3.0:1775764771.111545:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000009e8ce381/0x36f9bdc9e41ca973 lrc: 4/0,0 mode: PW/PW res: [0x240000403:0x50:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee327 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 129 lru_type: 1 00000020:00200000:3.0:1775764771.111552:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1775764771.111561:0:19724:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2af:0x0] updating i_size 260096 i_blocks 1 00000020:00010000:3.0:1775764771.111564:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_release():245 00000020:00010000:3.0:1775764771.111565:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.111567:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d5cb0 (ffff8f9970314758/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.111570:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764771.111572:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d5cb0 (ffff8f9970314e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764771.111586:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2af:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.111628:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.111633:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.111636:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:3.0:1775764771.111640:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00010000:00010000:0.0:1775764771.111642:0:19881:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966bb0800/0x36f9bdc9e421649d lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2ab:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e583b expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 8 lru_type: 0 00000080:00200000:3.0:1775764771.111643:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=16 00000080:00200000:3.0:1775764771.111645:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.111649:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00010000:00010000:0.0:1775764771.111651:0:19881:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966bb0800/0x36f9bdc9e421649d lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2ab:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e583b expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 8 lru_type: 0 00000080:00200000:3.0:1775764771.111651:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.111654:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=WORDPRO, flags=16 00000080:00200000:3.0:1775764771.111656:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2a9:0x0](ffff8f9952da6c08), inode mode 41c0 mask 1 00000080:00010000:0.0:1775764771.111658:0:19881:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421649d from it ffffafbac1f2fd48 00010000:00010000:0.0:1775764771.111661:0:19881:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966bb0800/0x36f9bdc9e421649d lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2ab:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5e583b expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 8 lru_type: 0 00000080:00200000:3.0:1775764771.111661:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=NEWS1_1.LWP, flags=0 00000080:00200000:3.0:1775764771.111667:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2a9:0x0](ffff8f9952da6c08), inode mode 41c0 mask 3 00010000:00010000:0.0:1775764771.111668:0:19881:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966bb0800/0x36f9bdc9e421649d lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ab:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e583b expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 9 lru_type: 1 00000080:00200000:3.0:1775764771.111672:0:19724:0:(namei.c:2395:ll_unlink()) VFS Op:name=NEWS1_1.LWP, dir=[0x200000401:0x2a9:0x0](ffff8f9952da6c08) 00010000:00010000:3.0:1775764771.111707:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480400/0x36f9bdc9e421c252 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f8b70 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00010000:00010000:3.0:1775764771.111721:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d8600/0x36f9bdc9e41ca8a8 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2a9:0x0].0x0 bits 0x5/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ee113 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00010000:00010000:3.0:1775764771.111730:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970abe800/0x36f9bdc9e41ca8f5 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x11/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ee1ec expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:3.0:1775764771.111767:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971227200/0x36f9bdc9e42168e8 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2af:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5e6618 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764771.111788:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2af:0x0]: losing layout lock 00010000:00010000:3.0:1775764771.111826:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971227200/0x36f9bdc9e42168e8 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2af:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5e6618 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764771.111834:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480400/0x36f9bdc9e421c252 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8b70 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00010000:00010000:3.0:1775764771.111840:0:19724:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1775764771.111843:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971227200/0x36f9bdc9e42168e8 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2af:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5e6618 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764771.111861:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480400/0x36f9bdc9e421c252 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8b70 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00000020:00100000:3.0:1775764771.111877:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 2 is allocated opc 36, max 7 00000100:00100000:3.0:1775764771.111888:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970681f80 x1862022966937728/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764771.111911:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970681f80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966937728:192.168.202.146@tcp:36:dbench.0 00000100:00100000:3.0:1775764771.111994:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99707ef000 going to sleep for 11 seconds 00000080:00200000:0.0:1775764771.114083:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.114089:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.114092:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:0.0:1775764771.114096:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.114098:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=16 00000080:00200000:0.0:1775764771.114100:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.114102:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:0.0:1775764771.114104:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.114106:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PWRPNT, flags=16 00000080:00200000:0.0:1775764771.114108:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x285:0x0](ffff8f9952da3a88), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.114113:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PCBENCHM.PPT, flags=0 00000080:00200000:0.0:1775764771.114115:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x285:0x0](ffff8f9952da3a88), inode mode 41c0 mask 3 00000080:00200000:0.0:1775764771.114119:0:19723:0:(namei.c:2395:ll_unlink()) VFS Op:name=PCBENCHM.PPT, dir=[0x200000401:0x285:0x0](ffff8f9952da3a88) 00010000:00010000:0.0:1775764771.114168:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c489 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x286:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f9ac7 expref: -99 pid: 19879 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1775764771.114189:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x286:0x0]: losing layout lock 00010000:00010000:0.0:1775764771.114206:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925a00/0x36f9bdc9e4218d85 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x286:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5ee5c3 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764771.114222:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x286:0x0]: losing layout lock 00010000:00010000:0.0:1775764771.114231:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a44400/0x36f9bdc9e41cb33d lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x286:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e4efcbf expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 12 lru_type: 1 00000080:00200000:0.0:1775764771.114251:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x286:0x0]: losing layout lock 00010000:00010000:0.0:1775764771.114282:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c489 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x286:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f9ac7 expref: -99 pid: 19879 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764771.114290:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925a00/0x36f9bdc9e4218d85 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x286:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5ee5c3 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764771.114295:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a44400/0x36f9bdc9e41cb33d lrc: 1/0,0 mode: --/CR res: [0x200000401:0x286:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e4efcbf expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 12 lru_type: 1 00010000:00010000:0.0:1775764771.114300:0:19723:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:0.0:1775764771.114303:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c489 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x286:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f9ac7 expref: -99 pid: 19879 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764771.114312:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925a00/0x36f9bdc9e4218d85 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x286:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5ee5c3 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764771.114320:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a44400/0x36f9bdc9e41cb33d lrc: 0/0,0 mode: --/CR res: [0x200000401:0x286:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e4efcbf expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 12 lru_type: 1 00000020:00100000:0.0:1775764771.114340:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1775764771.114350:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f0ab800 x1862022966937984/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 504/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764771.114370:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0ab800 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966937984:192.168.202.146@tcp:36:dbench.0 00000100:00100000:0.0:1775764771.114449:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f3c0 going to sleep for 11 seconds 00000100:00100000:3.0:1775764771.117492:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970681f80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966937728:192.168.202.146@tcp:36:dbench.0 00000080:00200000:3.0:1775764771.117563:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2af:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1775764771.117573:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764771.117589:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:3.0:1775764771.117590:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:3.0:1775764771.117632:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2af:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764771.118463:0:19724:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2af:0x0](ffff8f9952d8b248) 00000008:00010000:0.0:1775764771.118518:0:19724:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 1 [cma]time 1775764055 1775764055 0 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000009e8ce381/0x36f9bdc9e41ca973 lrc: 1/0,0 mode: PW/PW res: [0x240000403:0x50:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee327 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 129 lru_type: 1 00000080:00200000:0.0:1775764771.118630:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.118637:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.118644:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:0.0:1775764771.118649:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.118653:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:0.0:1775764771.118656:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.118659:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:0.0:1775764771.118661:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.118664:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=WORDPRO, flags=17 00000080:00200000:0.0:1775764771.118667:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2a9:0x0](ffff8f9952da6c08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.118670:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=RESULTS.XLS, flags=1 00010000:00010000:0.0:1775764771.118687:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fce00/0x36f9bdc9e421665d lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2ad:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5e5e07 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00010000:0.0:1775764771.118698:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421665d from it ffffafbac1c5baf8 00010000:00010000:0.0:1775764771.118702:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fce00/0x36f9bdc9e421665d lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2ad:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5e5e07 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:0.0:1775764771.118711:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fce00/0x36f9bdc9e421665d lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ad:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e5e07 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00000080:00200008:0.0:1775764771.118765:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file RESULTS.XLS:[0x200000401:0x2ad:0x0](ffff8f9952d16c08), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764771.118771:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2ad:0x0](ffff8f9952d16c08),name=RESULTS.XLS 00000002:00010000:0.0:1775764771.118788:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2ad:0x0]) in obj [0x200000401:0x2a9:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764771.118798:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fce00/0x36f9bdc9e421665d lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2ad:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e5e07 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:0.0:1775764771.118809:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fce00/0x36f9bdc9e421665d lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2ad:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e5e07 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00000080:00200000:0.0:1775764771.118817:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2a9:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764771.118823:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2ad:0x0](ffff8f9952d16c08) 00000080:00010000:0.0:1775764771.118826:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2ad:0x0](ffff8f9952d16c08) for lock 0x36f9bdc9e421665d 00000080:00010000:0.0:1775764771.118831:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421665d from it ffffafbac1c5bcb8 00010000:00010000:0.0:1775764771.118834:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fce00/0x36f9bdc9e421665d lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2ad:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5e5e07 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:0.0:1775764771.118841:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fce00/0x36f9bdc9e421665d lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ad:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e5e07 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 7 lru_type: 0 00000080:00200000:0.0:1775764771.118864:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2ad:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1775764771.118871:0:19724:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2ad:0x0] 00000008:00010000:0.0:1775764771.118892:0:19724:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1775764771.118896:0:19724:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0df830, osc lock ffff8f996686ee40, flags 0x400001002 00010000:00010000:0.0:1775764771.118911:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003557a767/0x36f9bdc9e41ca92d lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x4f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee27f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 149 lru_type: 1 00010000:00010000:0.0:1775764771.118922:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003557a767/0x36f9bdc9e41ca92d lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x4f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee27f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 149 lru_type: 1 00000020:00200000:0.0:1775764771.119237:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1775764771.119241:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003557a767/0x36f9bdc9e41ca92d lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x4f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ee27f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 149 lru_type: 1 00010000:00010000:0.0:1775764771.119255:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003557a767/0x36f9bdc9e41ca92d lrc: 4/0,0 mode: PW/PW res: [0x240000403:0x4f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee27f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 150 lru_type: 1 00000020:00200000:0.0:1775764771.119266:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1775764771.119275:0:19724:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2ad:0x0] updating i_size 260096 i_blocks 1 00000020:00010000:0.0:1775764771.119278:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970b8b908 (ffff8f9966dc2f38/0) at cl_lock_release():245 00000020:00010000:0.0:1775764771.119280:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970b8b908 (ffff8f9966dc2f38/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764771.119283:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0df830 (ffff8f9966dc2998/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764771.119286:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970b8b908 (ffff8f9966dc2f38/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764771.119288:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0df830 (ffff8f9966dc2f38/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764771.119303:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2ad:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764771.119322:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.119325:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.119328:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:0.0:1775764771.119331:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.119334:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=16 00000080:00200000:0.0:1775764771.119336:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.119338:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:0.0:1775764771.119341:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.119343:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=WORDPRO, flags=16 00000080:00200000:0.0:1775764771.119345:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2a9:0x0](ffff8f9952da6c08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.119349:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=RESULTS.XLS, flags=0 00000080:00200000:0.0:1775764771.119351:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2a9:0x0](ffff8f9952da6c08), inode mode 41c0 mask 3 00000080:00200000:0.0:1775764771.119355:0:19724:0:(namei.c:2395:ll_unlink()) VFS Op:name=RESULTS.XLS, dir=[0x200000401:0x2a9:0x0](ffff8f9952da6c08) 00010000:00010000:0.0:1775764771.119487:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fce00/0x36f9bdc9e421665d lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ad:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5e5e07 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 7 lru_type: 0 00000080:00200000:0.0:1775764771.119518:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2ad:0x0]: losing layout lock 00010000:00010000:0.0:1775764771.119559:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fce00/0x36f9bdc9e421665d lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2ad:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5e5e07 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 7 lru_type: 0 00010000:00010000:0.0:1775764771.119567:0:19724:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764771.119570:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fce00/0x36f9bdc9e421665d lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2ad:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5e5e07 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 7 lru_type: 0 00000020:00100000:0.0:1775764771.119592:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1775764771.119599:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f0a9180 x1862022966938240/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764771.119667:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0a9180 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966938240:192.168.202.146@tcp:36:dbench.0 00000100:00100000:0.0:1775764771.119760:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f3c0 going to sleep for 11 seconds 00000100:00100000:0.0:1775764771.123509:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0a9180 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966938240:192.168.202.146@tcp:36:dbench.0 00000080:00200000:0.0:1775764771.123582:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2ad:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764771.123595:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1775764771.123631:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:0.0:1775764771.123633:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:0.0:1775764771.123666:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2ad:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764771.124072:0:19724:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2ad:0x0](ffff8f9952d16c08) 00000008:00010000:0.0:1775764771.124194:0:19724:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 1 [cma]time 1775764055 1775764055 0 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003557a767/0x36f9bdc9e41ca92d lrc: 1/0,0 mode: PW/PW res: [0x240000403:0x4f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee27f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 150 lru_type: 1 00000080:00200000:0.0:1775764771.124281:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.124289:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.124295:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:0.0:1775764771.124301:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.124304:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:0.0:1775764771.124307:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.124311:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:0.0:1775764771.124314:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.124317:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=WORDPRO, flags=17 00000080:00200000:0.0:1775764771.124319:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2a9:0x0](ffff8f9952da6c08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.124323:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=BENCHS.LWP, flags=1 00010000:00010000:0.0:1775764771.124343:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966bb0800/0x36f9bdc9e421649d lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2ab:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5e583b expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 9 lru_type: 1 00000080:00010000:0.0:1775764771.124354:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421649d from it ffffafbac1c5baf8 00010000:00010000:0.0:1775764771.124358:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966bb0800/0x36f9bdc9e421649d lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2ab:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5e583b expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 9 lru_type: 1 00010000:00010000:0.0:1775764771.124366:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966bb0800/0x36f9bdc9e421649d lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ab:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e583b expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 10 lru_type: 1 00000080:00200008:0.0:1775764771.124409:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file BENCHS.LWP:[0x200000401:0x2ab:0x0](ffff8f9952d163c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764771.124415:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2ab:0x0](ffff8f9952d163c8),name=BENCHS.LWP 00000002:00010000:0.0:1775764771.124432:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2ab:0x0]) in obj [0x200000401:0x2a9:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764771.124442:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966bb0800/0x36f9bdc9e421649d lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2ab:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e583b expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 10 lru_type: 1 00010000:00010000:0.0:1775764771.124453:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966bb0800/0x36f9bdc9e421649d lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2ab:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e583b expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 10 lru_type: 1 00000080:00200000:0.0:1775764771.124462:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2a9:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764771.124468:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2ab:0x0](ffff8f9952d163c8) 00000080:00010000:0.0:1775764771.124472:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2ab:0x0](ffff8f9952d163c8) for lock 0x36f9bdc9e421649d 00000080:00010000:0.0:1775764771.124476:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421649d from it ffffafbac1c5bcb8 00010000:00010000:0.0:1775764771.124478:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966bb0800/0x36f9bdc9e421649d lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2ab:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5e583b expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 10 lru_type: 1 00010000:00010000:0.0:1775764771.124486:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966bb0800/0x36f9bdc9e421649d lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ab:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e583b expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 11 lru_type: 1 00000080:00200000:0.0:1775764771.124508:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2ab:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1775764771.124515:0:19724:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2ab:0x0] 00000008:00010000:0.0:1775764771.124533:0:19724:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1775764771.124536:0:19724:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0df530, osc lock ffff8f996686ee40, flags 0x400001002 00010000:00010000:0.0:1775764771.124550:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001e28c615/0x36f9bdc9e41ca8d9 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x4e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee19f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 71 lru_type: 1 00010000:00010000:0.0:1775764771.124560:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001e28c615/0x36f9bdc9e41ca8d9 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x4e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee19f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 71 lru_type: 1 00000020:00200000:0.0:1775764771.124571:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1775764771.124574:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001e28c615/0x36f9bdc9e41ca8d9 lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x4e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ee19f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 71 lru_type: 1 00010000:00010000:0.0:1775764771.124583:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001e28c615/0x36f9bdc9e41ca8d9 lrc: 4/0,0 mode: PW/PW res: [0x240000403:0x4e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee19f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 72 lru_type: 1 00000020:00200000:0.0:1775764771.124593:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1775764771.124601:0:19724:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2ab:0x0] updating i_size 260096 i_blocks 1 00000020:00010000:0.0:1775764771.124605:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970b8b908 (ffff8f9966dc2f38/0) at cl_lock_release():245 00000020:00010000:0.0:1775764771.124608:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970b8b908 (ffff8f9966dc2f38/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764771.124611:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0df530 (ffff8f9966dc2998/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764771.124625:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970b8b908 (ffff8f9966dc2f38/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764771.124629:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0df530 (ffff8f9966dc2f38/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764771.124643:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2ab:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764771.124662:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.124665:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.124668:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:0.0:1775764771.124671:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.124673:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=16 00000080:00200000:0.0:1775764771.124675:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.124677:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:0.0:1775764771.124679:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.124682:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=WORDPRO, flags=16 00000080:00200000:0.0:1775764771.124684:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2a9:0x0](ffff8f9952da6c08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.124688:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=BENCHS.LWP, flags=0 00000080:00200000:0.0:1775764771.124691:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2a9:0x0](ffff8f9952da6c08), inode mode 41c0 mask 3 00000080:00200000:0.0:1775764771.124695:0:19724:0:(namei.c:2395:ll_unlink()) VFS Op:name=BENCHS.LWP, dir=[0x200000401:0x2a9:0x0](ffff8f9952da6c08) 00010000:00010000:0.0:1775764771.124729:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966bb0800/0x36f9bdc9e421649d lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ab:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5e583b expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 11 lru_type: 1 00000080:00200000:0.0:1775764771.124771:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2ab:0x0]: losing layout lock 00010000:00010000:0.0:1775764771.124812:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966bb0800/0x36f9bdc9e421649d lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2ab:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5e583b expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 11 lru_type: 1 00010000:00010000:0.0:1775764771.124819:0:19724:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764771.124821:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966bb0800/0x36f9bdc9e421649d lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2ab:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5e583b expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 11 lru_type: 1 00000020:00100000:0.0:1775764771.124845:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1775764771.124854:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f0a9f80 x1862022966938752/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764771.124875:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0a9f80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966938752:192.168.202.146@tcp:36:dbench.0 00000100:00100000:0.0:1775764771.124954:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f3c0 going to sleep for 11 seconds 00000100:00100000:3.0:1775764771.142247:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966939264:192.168.202.146@tcp:101:dbench.0 00010000:00010000:3.0:1775764771.142263:0:19723:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98e00/0x36f9bdc9e421c49e lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2ea:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9c4f expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.142274:0:19723:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98e00/0x36f9bdc9e421c49e lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2ed:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9c4f expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.142281:0:19723:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98e00/0x36f9bdc9e421c49e lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2ed:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9c4f expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.142292:0:19723:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98e00/0x36f9bdc9e421c49e lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2ed:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9c4f expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.142302:0:19723:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98e00/0x36f9bdc9e421c49e lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2ed:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9c4f expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.142313:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98e00/0x36f9bdc9e421c49e lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2ed:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9c4f expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.142321:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98e00/0x36f9bdc9e421c49e lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2ed:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9c4f expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.142327:0:19723:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98e00/0x36f9bdc9e421c49e lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2ed:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9c4f expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764771.142337:0:19723:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f99703d0e00 x1862022966939264/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/928 e 0 to 0 dl 1775764787 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'dbench.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1775764771.142345:0:19723:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98e00/0x36f9bdc9e421c49e lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2ed:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9c4f expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:3.0:1775764771.142353:0:19723:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98e00/0x36f9bdc9e421c49e lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2ed:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9c4f expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.142362:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fd400/0x36f9bdc9e421a5e2 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2ed:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f3604 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.142369:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fd400/0x36f9bdc9e421a5e2 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2ed:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f3604 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.142373:0:19723:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98e00/0x36f9bdc9e421c49e lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2ed:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9c4f expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.142378:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98e00/0x36f9bdc9e421c49e lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2ed:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x2aac6a50e5f9c4f expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.143934:0:19723:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98e00/0x36f9bdc9e421c49e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ed:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x2aac6a50e5f9c4f expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.144959:0:19723:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000080:00200000:1.0:1775764771.144969:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2ea:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1775764771.145042:0:19723:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x2ed:0x0](ffff8f9952dd0908): rc = 0 00000080:00200000:1.0:1775764771.145056:0:19723:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x2ed:0x0]: layout version change: 3 -> 3 00000080:00010000:1.0:1775764771.145062:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2ed:0x0](ffff8f9952dd0908) for lock 0x36f9bdc9e421a5e2 00000080:00010000:1.0:1775764771.145071:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2ed:0x0](ffff8f9952dd0908) 00000080:00010000:1.0:1775764771.145073:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421a5e2 from it ffffafbac1c7bb20 00010000:00010000:1.0:1775764771.145077:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fd400/0x36f9bdc9e421a5e2 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2ed:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f3604 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764771.145084:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fd400/0x36f9bdc9e421a5e2 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ed:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f3604 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200008:1.0:1775764771.145128:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file MEDIUM.FIL:[0x200000401:0x2ed:0x0](ffff8f9952dd0908), request_mask 2047, flags 0, foreign 0 00010000:00010000:3.0:1775764771.145188:0:11138:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1775764771.145193:0:11138:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98e00/0x36f9bdc9e421c49e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ed:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f9c4f expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.145198:0:11138:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970a98e00 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:3.0:1775764771.145201:0:11138:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98e00/0x36f9bdc9e421c49e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ed:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f9c4f expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.145207:0:11138:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98e00/0x36f9bdc9e421c49e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ed:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f9c4f expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.145212:0:11138:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98e00/0x36f9bdc9e421c49e lrc: 3/0,0 mode: PR/PR res: [0x200000401:0x2ed:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f9c4f expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.145227:0:11138:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fd400/0x36f9bdc9e421a5e2 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ed:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f3604 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764771.145253:0:11138:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98e00/0x36f9bdc9e421c49e lrc: 2/0,0 mode: --/PR res: [0x200000401:0x2ed:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9c4f expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.145256:0:11138:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764771.145275:0:11138:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98e00/0x36f9bdc9e421c49e lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2ed:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9c4f expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.145280:0:11138:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98e00/0x36f9bdc9e421c49e lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2ed:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9c4f expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.145288:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764771.145289:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764771.145290:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764771.145302:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970683800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966939520:192.168.202.146@tcp:103:ldlm_bl.0 00000080:00200000:1.0:1775764771.145950:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2ed:0x0](ffff8f9952dd0908),name=MEDIUM.FIL 00000002:00010000:1.0:1775764771.145966:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2ed:0x0]) in obj [0x200000401:0x2ea:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764771.145973:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fd400/0x36f9bdc9e421a5e2 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2ed:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f3604 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764771.145979:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fd400/0x36f9bdc9e421a5e2 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2ed:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f3604 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764771.145983:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2ea:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764771.145987:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2ed:0x0](ffff8f9952dd0908) 00000080:00010000:1.0:1775764771.145989:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2ed:0x0](ffff8f9952dd0908) for lock 0x36f9bdc9e421a5e2 00000080:00010000:1.0:1775764771.145992:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421a5e2 from it ffffafbac1c7bcb8 00010000:00010000:1.0:1775764771.145994:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fd400/0x36f9bdc9e421a5e2 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2ed:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f3604 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764771.146000:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fd400/0x36f9bdc9e421a5e2 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ed:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f3604 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764771.146023:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2ed:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764771.146031:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2ed:0x0] 00000008:00010000:1.0:1775764771.146052:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764771.146054:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f1c7430, osc lock ffff8f995875db50, flags 0x400001002 00010000:00010000:1.0:1775764771.146067:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000b2fa2735/0x36f9bdc9e41cb225 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x6b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef99a expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764771.146079:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000b2fa2735/0x36f9bdc9e41cb225 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x6b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef99a expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00200000:1.0:1775764771.146086:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764771.146088:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000b2fa2735/0x36f9bdc9e41cb225 lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x6b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ef99a expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764771.146093:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000b2fa2735/0x36f9bdc9e41cb225 lrc: 4/0,0 mode: PW/PW res: [0x240000403:0x6b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef99a expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00200000:1.0:1775764771.146099:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764771.146107:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2ed:0x0] updating i_size 1048576 i_blocks 2051 00000020:00010000:1.0:1775764771.146109:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_release():245 00000020:00010000:1.0:1775764771.146111:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.146114:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f1c7430 (ffff8f99705b8ea8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.146116:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705f96d0 (ffff8f99705b8128/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764771.146118:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f1c7430 (ffff8f99705b8128/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764771.146133:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2ed:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764771.146162:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.146164:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.146168:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:1.0:1775764771.146171:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.146174:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=16 00000080:00200000:1.0:1775764771.146176:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764771.146178:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:1.0:1775764771.146181:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764771.146183:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=SEED, flags=16 00000080:00200000:1.0:1775764771.146184:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2ea:0x0](ffff8f9952dd7448), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764771.146188:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=MEDIUM.FIL, flags=0 00000080:00200000:1.0:1775764771.146193:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2ea:0x0](ffff8f9952dd7448), inode mode 41c0 mask 3 00000080:00200000:1.0:1775764771.146196:0:19723:0:(namei.c:2395:ll_unlink()) VFS Op:name=MEDIUM.FIL, dir=[0x200000401:0x2ea:0x0](ffff8f9952dd7448) 00010000:00010000:1.0:1775764771.146219:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49000/0x36f9bdc9e41cf8e2 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ea:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e4fd7be expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00010000:00010000:1.0:1775764771.146227:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4400/0x36f9bdc9e41cb1b5 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2ea:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ef87b expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764771.146280:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fd400/0x36f9bdc9e421a5e2 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ed:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f3604 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764771.146292:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2ed:0x0]: losing layout lock 00010000:00010000:1.0:1775764771.146312:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fd400/0x36f9bdc9e421a5e2 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2ed:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f3604 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764771.146317:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49000/0x36f9bdc9e41cf8e2 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2ea:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4fd7be expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00010000:00010000:1.0:1775764771.146321:0:19723:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1775764771.146322:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fd400/0x36f9bdc9e421a5e2 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2ed:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f3604 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764771.146331:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49000/0x36f9bdc9e41cf8e2 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2ea:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4fd7be expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00000020:00100000:1.0:1775764771.146341:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1775764771.146348:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970b53800 x1862022966939776/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764771.146360:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970b53800 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966939776:192.168.202.146@tcp:36:dbench.0 00000100:00100000:1.0:1775764771.146406:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f997130db40 going to sleep for 11 seconds 00000100:00100000:0.0:1775764771.150381:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:3.0:1775764771.150446:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970683800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966939520:192.168.202.146@tcp:103:ldlm_bl.0 00000080:00200000:0.0:1775764771.151234:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2b3:0x0](ffff8f9952d17448),name=T1.XLS 00000002:00010000:0.0:1775764771.151252:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2b3:0x0]) in obj [0x200000401:0x2b1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764771.151266:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4600/0x36f9bdc9e4217fb6 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b3:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5eaff3 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764771.151276:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4600/0x36f9bdc9e4217fb6 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5eaff3 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764771.151283:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2b1:0x0] suppgids 0 0: rc 1 00010000:00010000:1.0:1775764771.151287:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000080:00010000:0.0:1775764771.151289:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2b3:0x0](ffff8f9952d17448) 00000080:00010000:0.0:1775764771.151291:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2b3:0x0](ffff8f9952d17448) for lock 0x36f9bdc9e4217fb6 00010000:00010000:1.0:1775764771.151294:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98400/0x36f9bdc9e421c4a5 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f9c6b expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1775764771.151295:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e4217fb6 from it ffffafbac1c5bcb8 00010000:00010000:0.0:1775764771.151298:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4600/0x36f9bdc9e4217fb6 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b3:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5eaff3 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764771.151305:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4600/0x36f9bdc9e4217fb6 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5eaff3 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764771.151306:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970a98400 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:1.0:1775764771.151310:0:5571:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98400/0x36f9bdc9e421c4a5 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b3:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f9c6b expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.151318:0:5571:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98400/0x36f9bdc9e421c4a5 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b3:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f9c6b expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.151326:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98400/0x36f9bdc9e421c4a5 lrc: 3/0,0 mode: PR/PR res: [0x200000401:0x2b3:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f9c6b expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1775764771.151338:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2b3:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00010000:00010000:1.0:1775764771.151340:0:5571:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4600/0x36f9bdc9e4217fb6 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5eaff3 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000001:00010000:0.0:1775764771.151348:0:19724:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2b3:0x0] 00010000:00010000:1.0:1775764771.151371:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98400/0x36f9bdc9e421c4a5 lrc: 2/0,0 mode: --/PR res: [0x200000401:0x2b3:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9c6b expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1775764771.151375:0:19724:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00010000:00010000:1.0:1775764771.151377:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00000008:00010000:0.0:1775764771.151378:0:19724:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0df530, osc lock ffff8f996686ee40, flags 0x400001002 00010000:00010000:0.0:1775764771.151393:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000018bd3f91/0x36f9bdc9e41ca9f8 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x51:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee4b6 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 103 lru_type: 1 00010000:00010000:0.0:1775764771.151404:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000018bd3f91/0x36f9bdc9e41ca9f8 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x51:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee4b6 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 103 lru_type: 1 00000020:00200000:0.0:1775764771.151415:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764771.151416:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98400/0x36f9bdc9e421c4a5 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2b3:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9c6b expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.151419:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000018bd3f91/0x36f9bdc9e41ca9f8 lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x51:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ee4b6 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 103 lru_type: 1 00010000:00010000:0.0:1775764771.151427:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000018bd3f91/0x36f9bdc9e41ca9f8 lrc: 4/0,0 mode: PW/PW res: [0x240000403:0x51:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee4b6 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 104 lru_type: 1 00010000:00010000:1.0:1775764771.151430:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98400/0x36f9bdc9e421c4a5 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2b3:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9c6b expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000020:00200000:0.0:1775764771.151437:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:1.0:1775764771.151441:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764771.151443:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1775764771.151444:0:19724:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2b3:0x0] updating i_size 260096 i_blocks 1 00010000:00010000:1.0:1775764771.151444:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00010000:0.0:1775764771.151447:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970b8b908 (ffff8f9966dc2f38/0) at cl_lock_release():245 00000020:00010000:0.0:1775764771.151450:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970b8b908 (ffff8f9966dc2f38/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764771.151452:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0df530 (ffff8f9966dc2998/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764771.151455:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970b8b908 (ffff8f9966dc2f38/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764771.151457:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0df530 (ffff8f9966dc2f38/1) at cl_lock_fini():72 00000100:00100000:1.0:1775764771.151457:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978358784 00000100:00100000:1.0:1775764771.151463:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978358784 00000080:00200000:0.0:1775764771.151474:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2b3:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:1.0:1775764771.151501:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24108 00000080:00200000:0.0:1775764771.151506:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000100:00100000:1.0:1775764771.151508:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0a8000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+5:9373:x1862022978358784:12345-192.168.202.146@tcp:105: 00000080:00200000:0.0:1775764771.151511:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.151516:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00010000:00010000:1.0:1775764771.151517:0:5561:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98000/0x36f9bdc9e421c4ac lrc: 5/1,0 mode: --/PR res: [0x200000401:0x2ea:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9c80 expref: -99 pid: 19883 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1775764771.151521:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.151524:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=16 00010000:00010000:1.0:1775764771.151526:0:5561:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98000/0x36f9bdc9e421c4ac lrc: 5/1,0 mode: --/PR res: [0x200000401:0x2ea:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9c80 expref: -99 pid: 19883 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1775764771.151527:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.151530:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:0.0:1775764771.151533:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00010000:00010000:1.0:1775764771.151535:0:5561:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98000/0x36f9bdc9e421c4ac lrc: 5/1,0 mode: --/PR res: [0x200000401:0x2ea:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9c80 expref: -99 pid: 19883 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1775764771.151536:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PM, flags=16 00000080:00200000:0.0:1775764771.151538:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b1:0x0](ffff8f9952d142c8), inode mode 41c0 mask 1 00010000:00010000:1.0:1775764771.151543:0:5561:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98000/0x36f9bdc9e421c4ac lrc: 5/1,0 mode: PR/PR res: [0x200000401:0x2ea:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x10080000000000 nid: local remote: 0x2aac6a50e5f9c80 expref: -99 pid: 19883 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1775764771.151544:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=T1.XLS, flags=0 00000080:00200000:0.0:1775764771.151550:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b1:0x0](ffff8f9952d142c8), inode mode 41c0 mask 3 00010000:00010000:1.0:1775764771.151550:0:5561:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98000/0x36f9bdc9e421c4ac lrc: 6/1,0 mode: PR/PR res: [0x200000401:0x2ea:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x10080000000000 nid: local remote: 0x2aac6a50e5f9c80 expref: -99 pid: 19883 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1775764771.151555:0:19724:0:(namei.c:2395:ll_unlink()) VFS Op:name=T1.XLS, dir=[0x200000401:0x2b1:0x0](ffff8f9952d142c8) 00010000:00010000:1.0:1775764771.151558:0:5561:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98000/0x36f9bdc9e421c4ac lrc: 6/1,0 mode: PR/PR res: [0x200000401:0x2ea:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x80000000000 nid: local remote: 0x2aac6a50e5f9c80 expref: -99 pid: 19883 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.151582:0:5561:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8f9970a98000) 00010000:00010000:0.0:1775764771.151587:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941481c00/0x36f9bdc9e421c259 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f8b8c expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:0.0:1775764771.151597:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970abf400/0x36f9bdc9e41ca9ce lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2b1:0x0].0x0 bits 0x5/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ee43f expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 60 lru_type: 1 00010000:00010000:0.0:1775764771.151606:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970abf800/0x36f9bdc9e41caa0d lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x11/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ee50a expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 0 00000100:00100000:1.0:1775764771.151659:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0a8000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+5:9373:x1862022978358784:12345-192.168.202.146@tcp:105: Request processed in 150us (1282us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764771.151667:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24108 00010000:00010000:0.0:1775764771.151682:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4600/0x36f9bdc9e4217fb6 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5eaff3 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000100:00100000:1.0:1775764771.151693:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970b51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966940032:192.168.202.146@tcp:103:ldlm_bl.0 00000080:00200000:0.0:1775764771.151701:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2b3:0x0]: losing layout lock 00010000:00010000:0.0:1775764771.151727:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4600/0x36f9bdc9e4217fb6 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2b3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5eaff3 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764771.151734:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941481c00/0x36f9bdc9e421c259 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8b8c expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:0.0:1775764771.151755:0:19724:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1775764771.151758:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4600/0x36f9bdc9e4217fb6 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2b3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5eaff3 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764771.151772:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941481c00/0x36f9bdc9e421c259 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8b8c expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000020:00100000:0.0:1775764771.151787:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 2 is allocated opc 36, max 7 00000100:00100000:0.0:1775764771.151794:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f0c8e00 x1862022966940288/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 488/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764771.151808:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c8e00 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966940288:192.168.202.146@tcp:36:dbench.0 00000100:00100000:0.0:1775764771.151832:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f6c0 going to sleep for 11 seconds 00000002:00010000:0.0:1775764771.151853:0:19885:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2b1:0x0]) in obj [0x200000401:0x2b1:0x0], intent: readdir flags 0 00010000:00010000:0.0:1775764771.151859:0:19885:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/689 (0 0) 00010000:00010000:0.0:1775764771.151880:0:19885:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c4b3 lrc: 3/1,0 mode: --/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 19885 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.151887:0:19885:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c4b3 lrc: 3/1,0 mode: --/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19885 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.151897:0:19885:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c4b3 lrc: 3/1,0 mode: --/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19885 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764771.151909:0:19885:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0ca680 pname:cluuid:pid:xid:nid:opc:job ll_sa_19724:3f83546c-01a9-4156-8a97-44c446245d3e:19885:1862022966940416:192.168.202.146@tcp:101:ll_sa.0 00000100:00100000:0.0:1775764771.152379:0:19885:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16ff00 going to sleep for 11 seconds 00000100:00100000:1.0:1775764771.155130:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970b51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966940032:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764771.155514:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970b53800 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966939776:192.168.202.146@tcp:36:dbench.0 00000080:00200000:1.0:1775764771.155589:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2ed:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764771.155602:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00100000:0.0:1775764771.159667:0:19885:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0ca680 pname:cluuid:pid:xid:nid:opc:job ll_sa_19724:3f83546c-01a9-4156-8a97-44c446245d3e:19885:1862022966940416:192.168.202.146@tcp:101:ll_sa.0 00010000:00010000:0.0:1775764771.159695:0:19885:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c4b3 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9ca3 expref: -99 pid: 19885 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.159712:0:19885:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c4b3 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9ca3 expref: -99 pid: 19885 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00020000:00200000:0.0:1775764771.160899:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:0.0:1775764771.160903:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:0.0:1775764771.160937:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2ed:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:3.0:1775764771.161659:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966940672:192.168.202.146@tcp:63:ll_sa.0 00000080:00200000:0.0:1775764771.162255:0:19723:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2ed:0x0](ffff8f9952dd0908) 00000008:00010000:0.0:1775764771.162298:0:19723:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 1048576 blocks 2051 [cma]time 1775764750 1775764750 0 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000b2fa2735/0x36f9bdc9e41cb225 lrc: 1/0,0 mode: PW/PW res: [0x240000403:0x6b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef99a expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764771.162385:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.162392:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.162399:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:0.0:1775764771.162405:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.162408:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:0.0:1775764771.162411:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.162414:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:0.0:1775764771.162417:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.162430:0:19723:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=SEED, dir=[0x200000401:0x26c:0x0](ffff8f9952db6c08), flags=17 00000080:00200000:0.0:1775764771.162435:0:19723:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=SEED, dir=[0x200000401:0x26c:0x0](ffff8f9952db6c08), intent=0 00000002:00010000:0.0:1775764771.162450:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: SEED,[0x0:0x0:0x0]) in obj [0x200000401:0x26c:0x0], intent: lookup flags 0 00010000:00010000:0.0:1775764771.162481:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c4c1 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.162489:0:19723:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c4c1 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.162502:0:19723:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c4c1 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764771.162518:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0cad80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966940800:192.168.202.146@tcp:101:dbench.0 00000100:00100000:0.0:1775764771.162583:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16ff00 going to sleep for 11 seconds 00000100:00100000:0.0:1775764771.169227:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764771.169272:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978358912 00000100:00100000:1.0:1775764771.169283:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978358912 00000100:00100000:1.0:1775764771.169332:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24109 00000100:00100000:1.0:1775764771.169343:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c9500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+5:5796:x1862022978358912:12345-192.168.202.146@tcp:105: 00010000:00010000:1.0:1775764771.169364:0:5562:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c4b3 lrc: 5/1,0 mode: --/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9ca3 expref: -99 pid: 19885 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.169378:0:5562:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c4b3 lrc: 5/1,0 mode: --/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9ca3 expref: -99 pid: 19885 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.169703:0:2382:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99a00/0x36f9bdc9e421c4ba lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2ea:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9ccd expref: -99 pid: 19883 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.169717:0:2382:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99a00/0x36f9bdc9e421c4ba lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2ee:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9ccd expref: -99 pid: 19883 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.169723:0:2382:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99a00/0x36f9bdc9e421c4ba lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2ee:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9ccd expref: -99 pid: 19883 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.169731:0:2382:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99a00/0x36f9bdc9e421c4ba lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2ee:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9ccd expref: -99 pid: 19883 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.169769:0:2382:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99a00/0x36f9bdc9e421c4ba lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2ee:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9ccd expref: -99 pid: 19883 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.169786:0:2382:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99a00/0x36f9bdc9e421c4ba lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2ee:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9ccd expref: -99 pid: 19883 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.169794:0:2382:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99a00/0x36f9bdc9e421c4ba lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2ee:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9ccd expref: -99 pid: 19883 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.169801:0:2382:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99a00/0x36f9bdc9e421c4ba lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2ee:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9ccd expref: -99 pid: 19883 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764771.169812:0:2382:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f99703d0700 x1862022966940672/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 672/968 e 0 to 0 dl 1775764787 ref 1 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:3.0:1775764771.169823:0:2382:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99a00/0x36f9bdc9e421c4ba lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2ee:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9ccd expref: -99 pid: 19883 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:3.0:1775764771.169835:0:2382:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c4ba from it ffff8f994f0cf988 00010000:00010000:3.0:1775764771.169839:0:2382:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99a00/0x36f9bdc9e421c4ba lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2ee:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9ccd expref: -99 pid: 19883 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.169848:0:2382:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99a00/0x36f9bdc9e421c4ba lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ee:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9ccd expref: -99 pid: 19883 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000100:00100000:3.0:1775764771.170668:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966940672:192.168.202.146@tcp:63:ll_sa.0 00010000:00010000:3.0:1775764771.170698:0:19780:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99a00/0x36f9bdc9e421c4ba lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2ee:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9ccd expref: -99 pid: 19883 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764771.170808:0:19780:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x2ee:0x0](ffff8f9952d90908): rc = 0 00000080:00200000:3.0:1775764771.170826:0:19780:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x2ee:0x0]: layout version change: 3 -> 3 00000080:00010000:3.0:1775764771.170835:0:19780:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2ee:0x0](ffff8f9952d90908) for lock 0x36f9bdc9e421c4ba 00000080:00010000:3.0:1775764771.170845:0:19780:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c4ba from it ffff8f994f0cf988 00010000:00010000:3.0:1775764771.170848:0:19780:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99a00/0x36f9bdc9e421c4ba lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2ee:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9ccd expref: -99 pid: 19883 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.170855:0:19780:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99a00/0x36f9bdc9e421c4ba lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ee:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9ccd expref: -99 pid: 19883 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764771.170895:0:19884:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2ee:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764771.170904:0:19884:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2ee:0x0] 00000008:00010000:3.0:1775764771.171140:0:19884:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:3.0:1775764771.171144:0:19884:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d5cb0, osc lock ffff8f9966ace4c0, flags 0x400041012 00010000:00010000:3.0:1775764771.171162:0:19884:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000b2d6c772/0x36f9bdc9e41cb272 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x6b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4efa6c expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764771.171175:0:19884:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000b2d6c772/0x36f9bdc9e41cb272 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x6b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4efa6c expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764771.171183:0:19884:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000b2d6c772/0x36f9bdc9e41cb272 lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x6b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4efa6c expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764771.171189:0:19884:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000b2d6c772/0x36f9bdc9e41cb272 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x6b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4efa6c expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764771.171203:0:19884:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_release():245 00000020:00010000:3.0:1775764771.171205:0:19884:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.171207:0:19884:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d5cb0 (ffff8f9970314758/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.171210:0:19884:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764771.171212:0:19884:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d5cb0 (ffff8f9970314e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764771.171226:0:19884:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2ee:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:1.0:1775764771.171879:0:5562:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c4b3 lrc: 5/1,0 mode: --/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9ca3 expref: -99 pid: 19885 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.171887:0:5562:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c4b3 lrc: 5/1,0 mode: PR/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x10080000000000 nid: local remote: 0x2aac6a50e5f9ca3 expref: -99 pid: 19885 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.171901:0:5562:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c4b3 lrc: 6/1,0 mode: PR/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x10080000000000 nid: local remote: 0x2aac6a50e5f9ca3 expref: -99 pid: 19885 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.171910:0:5562:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c4b3 lrc: 6/1,0 mode: PR/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x80000000000 nid: local remote: 0x2aac6a50e5f9ca3 expref: -99 pid: 19885 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.172045:0:5562:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8f99821ccc00) 00000100:00100000:1.0:1775764771.172114:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c9500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+5:5796:x1862022978358912:12345-192.168.202.146@tcp:105: Request processed in 2771us (2896us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764771.172125:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24109 00000100:00100000:0.0:1775764771.174249:0:19885:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c8380 pname:cluuid:pid:xid:nid:opc:job ll_sa_19724:3f83546c-01a9-4156-8a97-44c446245d3e:19885:1862022966941056:192.168.202.146@tcp:37:ll_sa.0 00000100:00100000:0.0:1775764771.174419:0:19885:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16fb40 going to sleep for 11 seconds 00000100:00100000:0.0:1775764771.177908:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c8e00 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966940288:192.168.202.146@tcp:36:dbench.0 00000080:00200000:0.0:1775764771.177977:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2b3:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764771.177987:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1775764771.178001:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:0.0:1775764771.178002:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:0.0:1775764771.178036:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2b3:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764771.178368:0:19724:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2b3:0x0](ffff8f9952d17448) 00000008:00010000:0.0:1775764771.178414:0:19724:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 1 [cma]time 1775764055 1775764055 0 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000018bd3f91/0x36f9bdc9e41ca9f8 lrc: 1/0,0 mode: PW/PW res: [0x240000403:0x51:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee4b6 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 104 lru_type: 1 00000080:00200000:0.0:1775764771.178494:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.178501:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.178507:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:0.0:1775764771.178513:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.178516:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:0.0:1775764771.178519:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.178522:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:0.0:1775764771.178526:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.178529:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PM, flags=17 00000080:00200000:0.0:1775764771.178531:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b1:0x0](ffff8f9952d142c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.178544:0:19724:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=MOVED.DOC, dir=[0x200000401:0x2b1:0x0](ffff8f9952d142c8), flags=1 00000080:00200000:0.0:1775764771.178547:0:19724:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=MOVED.DOC, dir=[0x200000401:0x2b1:0x0](ffff8f9952d142c8), intent=getattr 00000002:00010000:0.0:1775764771.178576:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: MOVED.DOC,[0x0:0x0:0x0]) in obj [0x200000401:0x2b1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764771.178627:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c4cf lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b1:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.178635:0:19724:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c4cf lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.178645:0:19724:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c4cf lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764771.178669:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0cb100 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966941184:192.168.202.146@tcp:101:dbench.0 00000100:00100000:0.0:1775764771.179002:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f6c0 going to sleep for 11 seconds 00000002:00010000:0.0:1775764771.179021:0:19723:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99600/0x36f9bdc9e421c4c8 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2eb:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9d05 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.179035:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081a00/0x36f9bdc9e421a5cd lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2eb:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f35be expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764771.179046:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081a00/0x36f9bdc9e421a5cd lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2eb:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f35be expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764771.179053:0:19723:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99600/0x36f9bdc9e421c4c8 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2eb:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9d05 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.179059:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99600/0x36f9bdc9e421c4c8 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2eb:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x2aac6a50e5f9d05 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.179065:0:19723:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99600/0x36f9bdc9e421c4c8 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2eb:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x2aac6a50e5f9d05 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.179125:0:19723:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000080:00200000:0.0:1775764771.179132:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2ea:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1775764771.179188:0:19723:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x2eb:0x0](ffff8f9952dd4b08): rc = 0 00000080:00200000:0.0:1775764771.179209:0:19723:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x2eb:0x0]: layout version change: 3 -> 3 00000080:00010000:0.0:1775764771.179218:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2eb:0x0](ffff8f9952dd4b08) for lock 0x36f9bdc9e421a5cd 00000080:00010000:0.0:1775764771.179228:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2eb:0x0](ffff8f9952dd4b08) 00000080:00010000:0.0:1775764771.179232:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421a5cd from it ffffafbac1c7bb20 00010000:00010000:0.0:1775764771.179235:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081a00/0x36f9bdc9e421a5cd lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2eb:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f35be expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764771.179245:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081a00/0x36f9bdc9e421a5cd lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2eb:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f35be expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200008:0.0:1775764771.179286:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file SMALL.FIL:[0x200000401:0x2eb:0x0](ffff8f9952dd4b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764771.179292:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2eb:0x0](ffff8f9952dd4b08),name=SMALL.FIL 00000002:00010000:0.0:1775764771.179302:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2eb:0x0]) in obj [0x200000401:0x2ea:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764771.179310:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081a00/0x36f9bdc9e421a5cd lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2eb:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f35be expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764771.179318:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081a00/0x36f9bdc9e421a5cd lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2eb:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f35be expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764771.179324:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2ea:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764771.179328:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2eb:0x0](ffff8f9952dd4b08) 00000080:00010000:0.0:1775764771.179332:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2eb:0x0](ffff8f9952dd4b08) for lock 0x36f9bdc9e421a5cd 00000080:00010000:0.0:1775764771.179335:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421a5cd from it ffffafbac1c7bcb8 00010000:00010000:0.0:1775764771.179338:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081a00/0x36f9bdc9e421a5cd lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2eb:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f35be expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764771.179344:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081a00/0x36f9bdc9e421a5cd lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2eb:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f35be expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1775764771.179366:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2eb:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1775764771.179373:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2eb:0x0] 00000008:00010000:0.0:1775764771.179402:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1775764771.179406:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0df530, osc lock ffff8f996686ee40, flags 0x400001002 00010000:00010000:0.0:1775764771.179421:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000996c885b/0x36f9bdc9e41cb1df lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x6a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef8e4 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764771.179431:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000996c885b/0x36f9bdc9e41cb1df lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x6a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef8e4 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00200000:0.0:1775764771.179441:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1775764771.179445:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000996c885b/0x36f9bdc9e41cb1df lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x6a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ef8e4 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764771.179453:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000996c885b/0x36f9bdc9e41cb1df lrc: 4/0,0 mode: PW/PW res: [0x240000403:0x6a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef8e4 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00200000:0.0:1775764771.179463:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1775764771.179474:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2eb:0x0] updating i_size 327680 i_blocks 1027 00000020:00010000:0.0:1775764771.179479:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970b8b908 (ffff8f9966dc2f38/0) at cl_lock_release():245 00000020:00010000:0.0:1775764771.179482:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970b8b908 (ffff8f9966dc2f38/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764771.179485:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0df530 (ffff8f9966dc2998/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764771.179488:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970b8b908 (ffff8f9966dc2f38/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764771.179491:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0df530 (ffff8f9966dc2f38/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764771.179507:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2eb:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764771.179535:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.179539:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.179543:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:0.0:1775764771.179547:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.179551:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=16 00000080:00200000:0.0:1775764771.179554:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.179557:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:0.0:1775764771.179559:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.179563:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=SEED, flags=16 00000080:00200000:0.0:1775764771.179565:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2ea:0x0](ffff8f9952dd7448), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.179569:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=SMALL.FIL, flags=0 00000080:00200000:0.0:1775764771.179572:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2ea:0x0](ffff8f9952dd7448), inode mode 41c0 mask 3 00000080:00200000:0.0:1775764771.179577:0:19723:0:(namei.c:2395:ll_unlink()) VFS Op:name=SMALL.FIL, dir=[0x200000401:0x2ea:0x0](ffff8f9952dd7448) 00010000:00010000:0.0:1775764771.179612:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98000/0x36f9bdc9e421c4ac lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ea:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f9c80 expref: -99 pid: 19883 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764771.179645:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99600/0x36f9bdc9e421c4c8 lrc: 3/0,0 mode: PR/PR res: [0x200000401:0x2eb:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f9d05 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1775764771.179663:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2eb:0x0]: losing layout lock 00010000:00010000:0.0:1775764771.179678:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081a00/0x36f9bdc9e421a5cd lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2eb:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f35be expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1775764771.179697:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2eb:0x0]: losing layout lock 00010000:00010000:0.0:1775764771.179723:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99600/0x36f9bdc9e421c4c8 lrc: 2/0,0 mode: --/PR res: [0x200000401:0x2eb:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9d05 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.179731:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081a00/0x36f9bdc9e421a5cd lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2eb:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f35be expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764771.180167:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98000/0x36f9bdc9e421c4ac lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2ea:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9c80 expref: -99 pid: 19883 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764771.180176:0:19723:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:0.0:1775764771.180179:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f081a00/0x36f9bdc9e421a5cd lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2eb:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f35be expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764771.180192:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98000/0x36f9bdc9e421c4ac lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2ea:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9c80 expref: -99 pid: 19883 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000020:00100000:0.0:1775764771.180208:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1775764771.180218:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f0caa00 x1862022966941440/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 504/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764771.180237:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0caa00 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966941440:192.168.202.146@tcp:36:dbench.0 00000100:00100000:0.0:1775764771.180289:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f000 going to sleep for 11 seconds 00000100:00100000:0.0:1775764771.183906:0:19885:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c8380 pname:cluuid:pid:xid:nid:opc:job ll_sa_19724:3f83546c-01a9-4156-8a97-44c446245d3e:19885:1862022966941056:192.168.202.146@tcp:37:ll_sa.0 00010000:00010000:0.0:1775764771.184446:0:19885:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c4b3 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9ca3 expref: -99 pid: 19885 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.184466:0:19885:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c4b3 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9ca3 expref: -99 pid: 19885 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764771.184535:0:19885:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c4d6 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b1:0x0].0x0 bits 0x0/0x0 rrc: 6 type: IBT flags: 0x10000000000000 pid: 19885 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.184540:0:19885:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c4d6 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2b1:0x0].0x0 bits 0x3/0x0 rrc: 6 type: IBT flags: 0x0 pid: 19885 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764771.184820:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0cb100 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966941184:192.168.202.146@tcp:101:dbench.0 00010000:00010000:3.0:1775764771.184832:0:19724:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c4cf lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2b1:0x0].0x0 bits 0x2/0x0 rrc: 6 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9d36 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.184840:0:19724:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c4cf lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2b6:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9d36 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.184845:0:19724:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c4cf lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2b6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9d36 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.184853:0:19724:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c4cf lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2b6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9d36 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.184859:0:19724:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c4cf lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2b6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9d36 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.184865:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c4cf lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2b6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9d36 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.184873:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c4cf lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2b6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9d36 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.184877:0:19724:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c4cf lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2b6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9d36 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764771.184883:0:19724:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994f0cb100 x1862022966941184/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/928 e 0 to 0 dl 1775764787 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'dbench.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1775764771.184890:0:19724:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c4cf lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9d36 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:3.0:1775764771.184898:0:19724:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c4cf lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9d36 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.184905:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971205e00/0x36f9bdc9e42187f1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5ecbde expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.184911:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971205e00/0x36f9bdc9e42187f1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5ecbde expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.184916:0:19724:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c4cf lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9d36 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.184919:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c4cf lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x2aac6a50e5f9d36 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.184926:0:19724:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c4cf lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x2aac6a50e5f9d36 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.184946:0:19724:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000080:00200000:3.0:1775764771.184951:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2b1:0x0] suppgids 0 -1: rc 0 00000080:00200000:3.0:1775764771.185019:0:19724:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x2b6:0x0](ffff8f9952d11988): rc = 0 00000080:00200000:3.0:1775764771.185038:0:19724:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x2b6:0x0]: layout version change: 3 -> 3 00000080:00010000:3.0:1775764771.185049:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2b6:0x0](ffff8f9952d11988) for lock 0x36f9bdc9e42187f1 00000080:00010000:3.0:1775764771.185059:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2b6:0x0](ffff8f9952d11988) 00000080:00010000:3.0:1775764771.185062:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e42187f1 from it ffffafbac1c5bb20 00010000:00010000:3.0:1775764771.185066:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971205e00/0x36f9bdc9e42187f1 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5ecbde expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.185074:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971205e00/0x36f9bdc9e42187f1 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5ecbde expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200008:3.0:1775764771.185104:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file MOVED.DOC:[0x200000401:0x2b6:0x0](ffff8f9952d11988), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764771.185109:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2b6:0x0](ffff8f9952d11988),name=MOVED.DOC 00000002:00010000:3.0:1775764771.185118:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2b6:0x0]) in obj [0x200000401:0x2b1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764771.185124:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971205e00/0x36f9bdc9e42187f1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5ecbde expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764771.185129:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971205e00/0x36f9bdc9e42187f1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5ecbde expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764771.185133:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2b1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764771.185136:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2b6:0x0](ffff8f9952d11988) 00000080:00010000:3.0:1775764771.185138:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2b6:0x0](ffff8f9952d11988) for lock 0x36f9bdc9e42187f1 00000080:00010000:3.0:1775764771.185140:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e42187f1 from it ffffafbac1c5bcb8 00010000:00010000:3.0:1775764771.185142:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971205e00/0x36f9bdc9e42187f1 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5ecbde expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764771.185146:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971205e00/0x36f9bdc9e42187f1 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5ecbde expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764771.185164:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2b6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764771.185172:0:19724:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2b6:0x0] 00000008:00010000:3.0:1775764771.185191:0:19724:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1775764771.185193:0:19724:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d5cb0, osc lock ffff8f9966ace4c0, flags 0x400001002 00010000:00010000:3.0:1775764771.185205:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000095a0503a/0x36f9bdc9e41caa7d lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x52:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee622 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:3.0:1775764771.185212:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000095a0503a/0x36f9bdc9e41caa7d lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x52:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee622 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000020:00200000:3.0:1775764771.185220:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1775764771.185223:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000095a0503a/0x36f9bdc9e41caa7d lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x52:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ee622 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:3.0:1775764771.185228:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000095a0503a/0x36f9bdc9e41caa7d lrc: 4/0,0 mode: PW/PW res: [0x280000403:0x52:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee622 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00000020:00200000:3.0:1775764771.185233:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1775764771.185240:0:19724:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2b6:0x0] updating i_size 260096 i_blocks 1 00000020:00010000:3.0:1775764771.185244:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_release():245 00000020:00010000:3.0:1775764771.185245:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.185247:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d5cb0 (ffff8f9970314758/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.185250:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764771.185251:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d5cb0 (ffff8f9970314e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764771.185262:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2b6:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.185293:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.185297:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.185302:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:3.0:1775764771.185306:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.185310:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=16 00000080:00200000:3.0:1775764771.185312:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.185315:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:3.0:1775764771.185318:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.185320:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PM, flags=16 00000080:00200000:3.0:1775764771.185322:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b1:0x0](ffff8f9952d142c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.185327:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=MOVED.DOC, flags=0 00000080:00200000:3.0:1775764771.185331:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b1:0x0](ffff8f9952d142c8), inode mode 41c0 mask 3 00000080:00200000:3.0:1775764771.185335:0:19724:0:(namei.c:2395:ll_unlink()) VFS Op:name=MOVED.DOC, dir=[0x200000401:0x2b1:0x0](ffff8f9952d142c8) 00010000:00010000:3.0:1775764771.185369:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c4b3 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x2/0x0 rrc: 6 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f9ca3 expref: -99 pid: 19885 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.185390:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c4cf lrc: 3/0,0 mode: PR/PR res: [0x200000401:0x2b6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f9d36 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000080:00200000:3.0:1775764771.185408:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2b6:0x0]: losing layout lock 00010000:00010000:3.0:1775764771.185422:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971205e00/0x36f9bdc9e42187f1 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5ecbde expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764771.185442:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2b6:0x0]: losing layout lock 00010000:00010000:3.0:1775764771.185466:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c4cf lrc: 2/0,0 mode: --/PR res: [0x200000401:0x2b6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9d36 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.185473:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971205e00/0x36f9bdc9e42187f1 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2b6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5ecbde expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764771.185479:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c4b3 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9ca3 expref: -99 pid: 19885 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.185485:0:19724:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:3.0:1775764771.185487:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971205e00/0x36f9bdc9e42187f1 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2b6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5ecbde expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764771.185499:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c4b3 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9ca3 expref: -99 pid: 19885 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000020:00100000:3.0:1775764771.185512:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 2 is allocated opc 36, max 7 00000100:00100000:3.0:1775764771.185517:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970683800 x1862022966941824/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 504/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764771.185531:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970683800 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966941824:192.168.202.146@tcp:36:dbench.0 00000100:00100000:3.0:1775764771.185592:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99707ef6c0 going to sleep for 11 seconds 00010000:00010000:0.0:1775764771.185816:0:11210:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1775764771.185821:0:11210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c4cf lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2b6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9d36 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.185826:0:11210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99821cc400 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:0.0:1775764771.185829:0:11210:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c4cf lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2b6:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9d36 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.185834:0:11210:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c4cf lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2b6:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9d36 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.185838:0:11210:0:(ldlm_request.c:1765:ldlm_cli_cancel()) ### lock is already being destroyed 00010000:00010000:0.0:1775764771.185840:0:11210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c4cf lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2b6:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9d36 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.185844:0:11210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c4cf lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2b6:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9d36 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.185854:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764771.185856:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764771.185857:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764771.186121:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966941568:192.168.202.146@tcp:63:ll_sa.0 00010000:00010000:0.0:1775764771.189371:0:2379:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c4d6 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2b1:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9d6e expref: -99 pid: 19885 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.189403:0:2379:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c4d6 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2b5:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9d6e expref: -99 pid: 19885 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.189411:0:2379:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c4d6 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2b5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9d6e expref: -99 pid: 19885 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.189423:0:2379:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c4d6 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2b5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9d6e expref: -99 pid: 19885 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.189432:0:2379:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c4d6 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2b5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9d6e expref: -99 pid: 19885 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.189443:0:2379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c4d6 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2b5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9d6e expref: -99 pid: 19885 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.189452:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c4d6 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2b5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9d6e expref: -99 pid: 19885 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.189460:0:2379:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c4d6 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2b5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9d6e expref: -99 pid: 19885 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764771.189474:0:2379:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994f0c8380 x1862022966941568/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 672/968 e 0 to 0 dl 1775764787 ref 1 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:0.0:1775764771.189486:0:2379:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c4d6 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9d6e expref: -99 pid: 19885 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:0.0:1775764771.189500:0:2379:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c4d6 from it ffff8f9966bf4d88 00010000:00010000:0.0:1775764771.189503:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c4d6 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9d6e expref: -99 pid: 19885 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.189512:0:2379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c4d6 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9d6e expref: -99 pid: 19885 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1775764771.189539:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966941568:192.168.202.146@tcp:63:ll_sa.0 00010000:00010000:0.0:1775764771.189568:0:131:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c4d6 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9d6e expref: -99 pid: 19885 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764771.189649:0:131:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x2b5:0x0](ffff8f9952d65348): rc = 0 00000080:00200000:0.0:1775764771.189670:0:131:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x2b5:0x0]: layout version change: 3 -> 3 00000080:00010000:0.0:1775764771.189680:0:131:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2b5:0x0](ffff8f9952d65348) for lock 0x36f9bdc9e421c4d6 00000080:00010000:0.0:1775764771.189699:0:131:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c4d6 from it ffff8f9966bf4d88 00010000:00010000:0.0:1775764771.189702:0:131:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c4d6 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9d6e expref: -99 pid: 19885 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764771.189710:0:131:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c4d6 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9d6e expref: -99 pid: 19885 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000100:00100000:3.0:1775764771.194979:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970683800 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966941824:192.168.202.146@tcp:36:dbench.0 00000080:00200000:3.0:1775764771.195061:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2b6:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1775764771.195073:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764771.195092:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:3.0:1775764771.195095:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:3.0:1775764771.195124:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2b6:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.196724:0:19724:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2b6:0x0](ffff8f9952d11988) 00000008:00010000:3.0:1775764771.196808:0:19724:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 1 [cma]time 1775764055 1775764055 0 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000095a0503a/0x36f9bdc9e41caa7d lrc: 1/0,0 mode: PW/PW res: [0x280000403:0x52:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee622 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00000080:00200000:3.0:1775764771.196895:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.196902:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.196908:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:3.0:1775764771.196913:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.196916:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:3.0:1775764771.196919:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.196922:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:3.0:1775764771.196924:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.196927:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PM, flags=17 00000080:00200000:3.0:1775764771.196929:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b1:0x0](ffff8f9952d142c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.196941:0:19724:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=EVENTS.DOC, dir=[0x200000401:0x2b1:0x0](ffff8f9952d142c8), flags=1 00000080:00200000:3.0:1775764771.197084:0:19724:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=EVENTS.DOC, dir=[0x200000401:0x2b1:0x0](ffff8f9952d142c8), intent=getattr 00010000:00010000:3.0:1775764771.197102:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c4d6 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9d6e expref: -99 pid: 19885 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:3.0:1775764771.197115:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c4d6 from it ffffafbac1c5b900 00010000:00010000:3.0:1775764771.197119:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c4d6 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9d6e expref: -99 pid: 19885 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764771.197127:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c4d6 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9d6e expref: -99 pid: 19885 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:3.0:1775764771.197166:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file EVENTS.DOC:[0x200000401:0x2b5:0x0](ffff8f9952d65348), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764771.197172:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2b5:0x0](ffff8f9952d65348),name=EVENTS.DOC 00000002:00010000:3.0:1775764771.197189:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2b5:0x0]) in obj [0x200000401:0x2b1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764771.197198:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3b8a00/0x36f9bdc9e421869a lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b5:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5ec77e expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.197210:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3b8a00/0x36f9bdc9e421869a lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5ec77e expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764771.197218:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2b1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764771.197223:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2b5:0x0](ffff8f9952d65348) 00000080:00010000:3.0:1775764771.197227:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2b5:0x0](ffff8f9952d65348) for lock 0x36f9bdc9e421869a 00000080:00010000:3.0:1775764771.197232:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421869a from it ffffafbac1c5bcb8 00010000:00010000:3.0:1775764771.197235:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3b8a00/0x36f9bdc9e421869a lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5ec77e expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.197242:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3b8a00/0x36f9bdc9e421869a lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5ec77e expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764771.197268:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2b5:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764771.197277:0:19724:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2b5:0x0] 00000008:00010000:3.0:1775764771.197610:0:19724:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1775764771.197622:0:19724:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d5cb0, osc lock ffff8f9966ace4c0, flags 0x400001002 00010000:00010000:3.0:1775764771.197640:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000095a27c5b/0x36f9bdc9e41caa3e lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x52:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee56c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 128 lru_type: 1 00010000:00010000:3.0:1775764771.197658:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000095a27c5b/0x36f9bdc9e41caa3e lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x52:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee56c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 128 lru_type: 1 00000020:00200000:3.0:1775764771.197669:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1775764771.197673:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000095a27c5b/0x36f9bdc9e41caa3e lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x52:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ee56c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 128 lru_type: 1 00010000:00010000:3.0:1775764771.197682:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000095a27c5b/0x36f9bdc9e41caa3e lrc: 4/0,0 mode: PW/PW res: [0x240000403:0x52:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee56c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 129 lru_type: 1 00000020:00200000:3.0:1775764771.197692:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1775764771.197700:0:19724:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2b5:0x0] updating i_size 260096 i_blocks 1 00000020:00010000:3.0:1775764771.197705:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_release():245 00000020:00010000:3.0:1775764771.197708:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.197710:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d5cb0 (ffff8f9970314758/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.197714:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764771.197717:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d5cb0 (ffff8f9970314e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764771.197736:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2b5:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.197785:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.197790:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.197793:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:3.0:1775764771.197796:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.197799:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=16 00000080:00200000:3.0:1775764771.197801:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.197804:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:3.0:1775764771.197807:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.197810:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PM, flags=16 00000080:00200000:3.0:1775764771.197812:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b1:0x0](ffff8f9952d142c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.197817:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=EVENTS.DOC, flags=0 00000080:00200000:3.0:1775764771.197820:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b1:0x0](ffff8f9952d142c8), inode mode 41c0 mask 3 00000080:00200000:3.0:1775764771.197825:0:19724:0:(namei.c:2395:ll_unlink()) VFS Op:name=EVENTS.DOC, dir=[0x200000401:0x2b1:0x0](ffff8f9952d142c8) 00010000:00010000:3.0:1775764771.197862:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c4d6 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b5:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f9d6e expref: -99 pid: 19885 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764771.197883:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2b5:0x0]: losing layout lock 00010000:00010000:3.0:1775764771.197901:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3b8a00/0x36f9bdc9e421869a lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b5:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5ec77e expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764771.197917:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2b5:0x0]: losing layout lock 00010000:00010000:3.0:1775764771.198538:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c4d6 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2b5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f9d6e expref: -99 pid: 19885 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764771.198547:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3b8a00/0x36f9bdc9e421869a lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2b5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5ec77e expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764771.198553:0:19724:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1775764771.198556:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c4d6 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2b5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f9d6e expref: -99 pid: 19885 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764771.198567:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3b8a00/0x36f9bdc9e421869a lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2b5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5ec77e expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000020:00100000:3.0:1775764771.198588:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1775764771.198595:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970681500 x1862022966942336/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764771.198629:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970681500 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966942336:192.168.202.146@tcp:36:dbench.0 00000100:00100000:3.0:1775764771.198697:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99707ef6c0 going to sleep for 11 seconds 00000100:00100000:3.0:1775764771.203833:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970681500 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966942336:192.168.202.146@tcp:36:dbench.0 00000080:00200000:3.0:1775764771.203892:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2b5:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1775764771.203903:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764771.203926:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:3.0:1775764771.203928:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:3.0:1775764771.204064:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2b5:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.204405:0:19724:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2b5:0x0](ffff8f9952d65348) 00000008:00010000:3.0:1775764771.204430:0:19724:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 1 [cma]time 1775764055 1775764055 0 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000095a27c5b/0x36f9bdc9e41caa3e lrc: 1/0,0 mode: PW/PW res: [0x240000403:0x52:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee56c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 129 lru_type: 1 00000080:00200000:3.0:1775764771.204478:0:19724:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2b1:0x0](ffff8f9952d142c8) pos/size18446744073709551614/7168 32bit_api 0 00000080:00200008:3.0:1775764771.204487:0:19724:0:(file.c:447:ll_file_release()) START file PM:[0x200000401:0x2b1:0x0](ffff8f9952d142c8), flags 304000 00010000:00010000:3.0:1775764771.204583:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970abf400/0x36f9bdc9e41ca9ce lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2b1:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ee43f expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 61 lru_type: 1 00000080:00200000:3.0:1775764771.204607:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.204930:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.204933:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:3.0:1775764771.204936:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.204939:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:3.0:1775764771.204953:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.204956:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:3.0:1775764771.204959:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.204962:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=COREL, flags=1 00010000:00010000:3.0:1775764771.204971:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c260 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8ba8 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00010000:3.0:1775764771.204978:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c260 from it ffffafbac1c5baf8 00010000:00010000:3.0:1775764771.204981:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c260 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8ba8 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764771.204988:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c260 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8ba8 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200008:3.0:1775764771.204998:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file COREL:[0x200000401:0x292:0x0](ffff8f9952d89148), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764771.205001:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x292:0x0](ffff8f9952d89148),name=COREL 00000002:00010000:3.0:1775764771.205008:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x292:0x0]) in obj [0x200000401:0x26b:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764771.205015:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c260 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8ba8 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764771.205023:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c260 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8ba8 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764771.205029:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26b:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764771.205033:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x292:0x0](ffff8f9952d89148) 00000080:00010000:3.0:1775764771.205035:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x292:0x0](ffff8f9952d89148) for lock 0x36f9bdc9e421c260 00000080:00010000:3.0:1775764771.205038:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c260 from it ffffafbac1c5bcb8 00010000:00010000:3.0:1775764771.205041:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c260 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8ba8 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764771.205048:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c260 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8ba8 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1775764771.205072:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.205076:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.205079:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=19 00000080:00200000:3.0:1775764771.205081:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.205084:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=19 00000080:00200000:3.0:1775764771.205086:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.205089:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=19 00000080:00200000:3.0:1775764771.205091:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.205093:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=COREL, flags=259 00000080:00200000:3.0:1775764771.205096:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x292:0x0](ffff8f9952d89148), inode mode 41c0 mask 44 00000080:00200008:3.0:1775764771.205099:0:19724:0:(file.c:1057:ll_file_open()) START file COREL:[0x200000401:0x292:0x0](ffff8f9952d89148), flags 304000 00000080:00200008:3.0:1775764771.205108:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file COREL:[0x200000401:0x292:0x0](ffff8f9952d89148), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764771.205111:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x292:0x0](ffff8f9952d89148),name=COREL 00000002:00010000:3.0:1775764771.205117:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x292:0x0]) in obj [0x200000401:0x26b:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764771.205123:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c260 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8ba8 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1775764771.205130:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c260 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8ba8 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1775764771.205135:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26b:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764771.205138:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x292:0x0](ffff8f9952d89148) 00000080:00010000:3.0:1775764771.205141:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x292:0x0](ffff8f9952d89148) for lock 0x36f9bdc9e421c260 00000080:00010000:3.0:1775764771.205143:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c260 from it ffffafbac1c5bcd8 00010000:00010000:3.0:1775764771.205146:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c260 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8ba8 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1775764771.205156:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c260 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8ba8 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1775764771.205167:0:19724:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x292:0x0](ffff8f9952d89148) pos/size0/7168 32bit_api 0 00000002:00010000:3.0:1775764771.205172:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x292:0x0]) in obj [0x200000401:0x292:0x0], intent: readdir flags 0 00010000:00010000:3.0:1775764771.205177:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c260 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8ba8 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1775764771.205183:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c260 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8ba8 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000100:00100000:3.0:1775764771.207023:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970681c00 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966942592:192.168.202.146@tcp:37:dbench.0 00000100:00100000:3.0:1775764771.207078:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99707ef6c0 going to sleep for 11 seconds 00000100:00100000:3.0:1775764771.218127:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970681c00 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966942592:192.168.202.146@tcp:37:dbench.0 00010000:00010000:3.0:1775764771.219120:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c260 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8ba8 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1775764771.219135:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9941480a00/0x36f9bdc9e421c260 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8ba8 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:0.0:1775764771.219419:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.219427:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.219433:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:0.0:1775764771.219439:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.219443:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:0.0:1775764771.219447:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.219450:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:0.0:1775764771.219452:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.219455:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=COREL, flags=17 00000080:00200000:0.0:1775764771.219458:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x292:0x0](ffff8f9952d89148), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.219472:0:19724:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=SIGN2.CPT, dir=[0x200000401:0x292:0x0](ffff8f9952d89148), flags=1 00000080:00200000:0.0:1775764771.219477:0:19724:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=SIGN2.CPT, dir=[0x200000401:0x292:0x0](ffff8f9952d89148), intent=getattr 00000002:00010000:0.0:1775764771.219499:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: SIGN2.CPT,[0x0:0x0:0x0]) in obj [0x200000401:0x292:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764771.219547:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421c4e4 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x292:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.219555:0:19724:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421c4e4 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x292:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.219566:0:19724:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421c4e4 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x292:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764771.219589:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c8380 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966942848:192.168.202.146@tcp:101:dbench.0 00000100:00100000:0.0:1775764771.219777:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16fb40 going to sleep for 11 seconds 00010000:00010000:1.0:1775764771.222524:0:5572:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764771.222531:0:5572:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99a00/0x36f9bdc9e421c4dd lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f9dc9 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.222539:0:5572:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970a99a00 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:1.0:1775764771.222544:0:5572:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99a00/0x36f9bdc9e421c4dd lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e4:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f9dc9 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.222552:0:5572:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99a00/0x36f9bdc9e421c4dd lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e4:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f9dc9 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.222560:0:5572:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99a00/0x36f9bdc9e421c4dd lrc: 3/0,0 mode: PR/PR res: [0x200000401:0x2e4:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f9dc9 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.222582:0:5572:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8da00/0x36f9bdc9e421c0a7 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8692 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764771.222612:0:5572:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99a00/0x36f9bdc9e421c4dd lrc: 2/0,0 mode: --/PR res: [0x200000401:0x2e4:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9dc9 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.222628:0:5572:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764771.222652:0:5572:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99a00/0x36f9bdc9e421c4dd lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2e4:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9dc9 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.222659:0:5572:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99a00/0x36f9bdc9e421c4dd lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2e4:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9dc9 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.222671:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764771.222673:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764771.222675:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764771.222691:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970b51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966942976:192.168.202.146@tcp:103:ldlm_bl.0 00000002:00010000:0.0:1775764771.225508:0:19887:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2bb:0x0]) in obj [0x200000401:0x2bb:0x0], intent: readdir flags 0 00010000:00010000:0.0:1775764771.225517:0:19887:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/699 (0 0) 00010000:00010000:0.0:1775764771.225541:0:19887:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c4eb lrc: 3/1,0 mode: --/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 19887 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.225549:0:19887:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c4eb lrc: 3/1,0 mode: --/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19887 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.225558:0:19887:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c4eb lrc: 3/1,0 mode: --/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19887 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764771.225573:0:19887:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c9180 pname:cluuid:pid:xid:nid:opc:job ll_sa_19723:3f83546c-01a9-4156-8a97-44c446245d3e:19887:1862022966943360:192.168.202.146@tcp:101:ll_sa.0 00000100:00100000:0.0:1775764771.225613:0:19887:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f540 going to sleep for 11 seconds 00000100:00100000:1.0:1775764771.226106:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970b51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966942976:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1775764771.227260:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764771.227267:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421c4e4 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x29f:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f9de5 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.227274:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99821cd000 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:1.0:1775764771.227278:0:19801:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421c4e4 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x29f:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f9de5 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.227291:0:19801:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421c4e4 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x29f:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5f9de5 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.227299:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421c4e4 lrc: 3/0,0 mode: PR/PR res: [0x200000401:0x29f:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f9de5 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.227313:0:19801:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8c200/0x36f9bdc9e42173f4 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x29f:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e88af expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764771.227341:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421c4e4 lrc: 2/0,0 mode: --/PR res: [0x200000401:0x29f:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9de5 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.227346:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764771.227363:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421c4e4 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x29f:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9de5 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.227370:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421c4e4 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x29f:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9de5 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.227381:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764771.227382:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764771.227384:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764771.227395:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970b51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966943488:192.168.202.146@tcp:103:ldlm_bl.0 00000002:00010000:3.0:1775764771.229659:0:19889:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x292:0x0]) in obj [0x200000401:0x292:0x0], intent: readdir flags 0 00010000:00010000:3.0:1775764771.229668:0:19889:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/658 (0 0) 00010000:00010000:3.0:1775764771.229692:0:19889:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5800/0x36f9bdc9e421c4f2 lrc: 3/1,0 mode: --/PR res: [0x200000401:0x292:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 19889 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.229700:0:19889:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5800/0x36f9bdc9e421c4f2 lrc: 3/1,0 mode: --/PR res: [0x200000401:0x292:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19889 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.229709:0:19889:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5800/0x36f9bdc9e421c4f2 lrc: 3/1,0 mode: --/PR res: [0x200000401:0x292:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19889 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764771.229723:0:19889:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970681c00 pname:cluuid:pid:xid:nid:opc:job ll_sa_19724:3f83546c-01a9-4156-8a97-44c446245d3e:19889:1862022966943872:192.168.202.146@tcp:101:ll_sa.0 00000100:00100000:3.0:1775764771.229793:0:19889:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99707ef6c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764771.235580:0:19887:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c9180 pname:cluuid:pid:xid:nid:opc:job ll_sa_19723:3f83546c-01a9-4156-8a97-44c446245d3e:19887:1862022966943360:192.168.202.146@tcp:101:ll_sa.0 00010000:00010000:1.0:1775764771.235607:0:19887:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c4eb lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9e01 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.236436:0:19887:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c4eb lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9e01 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764771.237190:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764771.237217:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978359040 00000100:00100000:1.0:1775764771.237225:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978359040 00000100:00100000:1.0:1775764771.237271:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24110 00000100:00100000:1.0:1775764771.237280:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970b53100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+5:5796:x1862022978359040:12345-192.168.202.146@tcp:105: 00010000:00010000:1.0:1775764771.237293:0:5561:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c4eb lrc: 5/1,0 mode: --/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9e01 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.237302:0:5561:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c4eb lrc: 5/1,0 mode: --/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9e01 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.237311:0:5561:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c4eb lrc: 5/1,0 mode: --/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9e01 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.237320:0:5561:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c4eb lrc: 5/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x10080000000000 nid: local remote: 0x2aac6a50e5f9e01 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.237330:0:5561:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c4eb lrc: 6/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x10080000000000 nid: local remote: 0x2aac6a50e5f9e01 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.237341:0:5561:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c4eb lrc: 6/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x80000000000 nid: local remote: 0x2aac6a50e5f9e01 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.237374:0:5561:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8f99821cc400) 00000100:00100000:1.0:1775764771.237432:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970b53100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+5:5796:x1862022978359040:12345-192.168.202.146@tcp:105: Request processed in 153us (245us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764771.237440:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24110 00000100:00100000:0.0:1775764771.238583:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970b51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966943488:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764771.251325:0:19889:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970681c00 pname:cluuid:pid:xid:nid:opc:job ll_sa_19724:3f83546c-01a9-4156-8a97-44c446245d3e:19889:1862022966943872:192.168.202.146@tcp:101:ll_sa.0 00010000:00010000:3.0:1775764771.251347:0:19889:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5800/0x36f9bdc9e421c4f2 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9e24 expref: -99 pid: 19889 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.251358:0:19889:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5800/0x36f9bdc9e421c4f2 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9e24 expref: -99 pid: 19889 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764771.251369:0:19889:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=4 disposition=0, status=0 req@ffff8f9970681c00 x1862022966943872/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 328/344 e 0 to 0 dl 1775764787 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:0 00000100:00100000:3.0:1775764771.251972:0:19889:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970681c00 pname:cluuid:pid:xid:nid:opc:job ll_sa_19724:3f83546c-01a9-4156-8a97-44c446245d3e:19889:1862022966944384:192.168.202.146@tcp:37:ll_sa.0 00000100:00100000:3.0:1775764771.252027:0:19889:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99707ef6c0 going to sleep for 11 seconds 00000100:00100000:3.0:1775764771.259641:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966944512:192.168.202.146@tcp:63:ll_sa.0 00010000:00010000:3.0:1775764771.259709:0:11138:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1775764771.259713:0:11138:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99e00/0x36f9bdc9e421c500 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2c2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9e5c expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.259719:0:11138:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970a99e00 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:3.0:1775764771.259721:0:11138:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99e00/0x36f9bdc9e421c500 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2c2:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9e5c expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.259730:0:11138:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99e00/0x36f9bdc9e421c500 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2c2:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9e5c expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.259735:0:11138:0:(ldlm_request.c:1765:ldlm_cli_cancel()) ### lock is already being destroyed 00010000:00010000:3.0:1775764771.259752:0:11138:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99e00/0x36f9bdc9e421c500 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2c2:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9e5c expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.259756:0:11138:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99e00/0x36f9bdc9e421c500 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2c2:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9e5c expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.259769:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764771.259770:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764771.259771:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764771.259795:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966944128:192.168.202.146@tcp:101:dbench.0 00010000:00010000:3.0:1775764771.259803:0:19724:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99600/0x36f9bdc9e421c4f9 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x292:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9e40 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.259810:0:19724:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99600/0x36f9bdc9e421c4f9 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x29d:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9e40 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.259814:0:19724:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99600/0x36f9bdc9e421c4f9 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x29d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9e40 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.259821:0:19724:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99600/0x36f9bdc9e421c4f9 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x29d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9e40 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.259826:0:19724:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99600/0x36f9bdc9e421c4f9 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x29d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9e40 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.259832:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99600/0x36f9bdc9e421c4f9 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x29d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9e40 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.259836:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99600/0x36f9bdc9e421c4f9 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x29d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9e40 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.259839:0:19724:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99600/0x36f9bdc9e421c4f9 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x29d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9e40 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764771.259845:0:19724:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f99703d0000 x1862022966944128/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/928 e 0 to 0 dl 1775764787 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'dbench.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1775764771.259851:0:19724:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99600/0x36f9bdc9e421c4f9 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x29d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9e40 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:3.0:1775764771.259858:0:19724:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99600/0x36f9bdc9e421c4f9 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x29d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9e40 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.259864:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49800/0x36f9bdc9e42173a7 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x29d:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e879e expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.259869:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49800/0x36f9bdc9e42173a7 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x29d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e879e expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.259873:0:19724:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99600/0x36f9bdc9e421c4f9 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x29d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9e40 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.259877:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99600/0x36f9bdc9e421c4f9 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x29d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x2aac6a50e5f9e40 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.259882:0:19724:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99600/0x36f9bdc9e421c4f9 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x29d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x2aac6a50e5f9e40 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.259890:0:19724:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000080:00200000:3.0:1775764771.259893:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x292:0x0] suppgids 0 -1: rc 0 00000080:00200000:3.0:1775764771.259936:0:19724:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x29d:0x0](ffff8f9952d8db88): rc = 0 00000080:00200000:3.0:1775764771.259952:0:19724:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x29d:0x0]: layout version change: 3 -> 3 00000080:00010000:3.0:1775764771.259960:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x29d:0x0](ffff8f9952d8db88) for lock 0x36f9bdc9e42173a7 00000080:00010000:3.0:1775764771.259969:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x29d:0x0](ffff8f9952d8db88) 00000080:00010000:3.0:1775764771.259972:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e42173a7 from it ffffafbac1c5bb20 00010000:00010000:3.0:1775764771.259975:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49800/0x36f9bdc9e42173a7 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x29d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5e879e expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.259982:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49800/0x36f9bdc9e42173a7 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x29d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e879e expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200008:3.0:1775764771.260010:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file TEAPOT2.CPT:[0x200000401:0x29d:0x0](ffff8f9952d8db88), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764771.260015:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x29d:0x0](ffff8f9952d8db88),name=TEAPOT2.CPT 00000002:00010000:3.0:1775764771.260024:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x29d:0x0]) in obj [0x200000401:0x292:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764771.260031:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49800/0x36f9bdc9e42173a7 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x29d:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e879e expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764771.260039:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49800/0x36f9bdc9e42173a7 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x29d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e879e expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764771.260044:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x292:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764771.260048:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x29d:0x0](ffff8f9952d8db88) 00000080:00010000:3.0:1775764771.260050:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x29d:0x0](ffff8f9952d8db88) for lock 0x36f9bdc9e42173a7 00000080:00010000:3.0:1775764771.260053:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e42173a7 from it ffffafbac1c5bcb8 00010000:00010000:3.0:1775764771.260056:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49800/0x36f9bdc9e42173a7 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x29d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5e879e expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764771.260061:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49800/0x36f9bdc9e42173a7 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x29d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e879e expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764771.260080:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x29d:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764771.260085:0:19724:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x29d:0x0] 00000008:00010000:3.0:1775764771.260104:0:19724:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1775764771.260107:0:19724:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d5cb0, osc lock ffff8f9966ace4c0, flags 0x400001002 00010000:00010000:3.0:1775764771.260120:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f01cb6c5/0x36f9bdc9e41ca6ef lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x48:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4edcb3 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 41 lru_type: 1 00010000:00010000:3.0:1775764771.260130:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f01cb6c5/0x36f9bdc9e41ca6ef lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x48:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4edcb3 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 41 lru_type: 1 00000020:00200000:3.0:1775764771.260139:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1775764771.260142:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f01cb6c5/0x36f9bdc9e41ca6ef lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x48:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4edcb3 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 41 lru_type: 1 00010000:00010000:3.0:1775764771.260149:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f01cb6c5/0x36f9bdc9e41ca6ef lrc: 4/0,0 mode: PW/PW res: [0x240000403:0x48:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4edcb3 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 42 lru_type: 1 00000020:00200000:3.0:1775764771.260159:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1775764771.260166:0:19724:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x29d:0x0] updating i_size 260096 i_blocks 1 00000020:00010000:3.0:1775764771.260169:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_release():245 00000020:00010000:3.0:1775764771.260171:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.260175:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d5cb0 (ffff8f9970314758/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.260178:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764771.260180:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d5cb0 (ffff8f9970314e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764771.260195:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x29d:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.260224:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.260228:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.260231:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:3.0:1775764771.260235:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.260239:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=16 00000080:00200000:3.0:1775764771.260241:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.260244:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:3.0:1775764771.260247:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.260250:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=COREL, flags=16 00000080:00200000:3.0:1775764771.260252:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x292:0x0](ffff8f9952d89148), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.260256:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=TEAPOT2.CPT, flags=0 00000080:00200000:3.0:1775764771.260258:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x292:0x0](ffff8f9952d89148), inode mode 41c0 mask 3 00000080:00200000:3.0:1775764771.260262:0:19724:0:(namei.c:2395:ll_unlink()) VFS Op:name=TEAPOT2.CPT, dir=[0x200000401:0x292:0x0](ffff8f9952d89148) 00010000:00010000:3.0:1775764771.260292:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99600/0x36f9bdc9e421c4f9 lrc: 3/0,0 mode: PR/PR res: [0x200000401:0x29d:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f9e40 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000080:00200000:3.0:1775764771.260308:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x29d:0x0]: losing layout lock 00010000:00010000:3.0:1775764771.260322:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49800/0x36f9bdc9e42173a7 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x29d:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5e879e expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764771.260337:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x29d:0x0]: losing layout lock 00010000:00010000:3.0:1775764771.260359:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99600/0x36f9bdc9e421c4f9 lrc: 2/0,0 mode: --/PR res: [0x200000401:0x29d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9e40 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.260368:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49800/0x36f9bdc9e42173a7 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x29d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5e879e expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764771.260371:0:19724:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1775764771.260373:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49800/0x36f9bdc9e42173a7 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x29d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5e879e expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000020:00100000:3.0:1775764771.260383:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 2 is allocated opc 36, max 7 00000100:00100000:3.0:1775764771.260387:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970681180 x1862022966945024/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764771.260399:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970681180 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966945024:192.168.202.146@tcp:36:dbench.0 00000100:00100000:3.0:1775764771.260427:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99707ef900 going to sleep for 11 seconds 00010000:00010000:1.0:1775764771.261480:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764771.261487:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99600/0x36f9bdc9e421c4f9 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x29d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9e40 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.261493:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970a99600 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:1.0:1775764771.261496:0:5571:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99600/0x36f9bdc9e421c4f9 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x29d:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9e40 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.261501:0:5571:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99600/0x36f9bdc9e421c4f9 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x29d:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9e40 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.261505:0:5571:0:(ldlm_request.c:1765:ldlm_cli_cancel()) ### lock is already being destroyed 00010000:00010000:1.0:1775764771.261507:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99600/0x36f9bdc9e421c4f9 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x29d:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9e40 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.261511:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99600/0x36f9bdc9e421c4f9 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x29d:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9e40 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.261524:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764771.261525:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764771.261526:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764771.266590:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764771.266919:0:5562:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1775764771.266946:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978359168 00000100:00100000:1.0:1775764771.266955:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978359168 00000100:00100000:1.0:1775764771.267007:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24111 00000100:00100000:1.0:1775764771.267017:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970e0a850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+5:5796:x1862022978359168:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764771.267036:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5800/0x36f9bdc9e421c4f2 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x2aac6a50e5f9e24 expref: -99 pid: 19889 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.267086:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764771.267092:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970e0a850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+5:5796:x1862022978359168:12345-192.168.202.146@tcp:104: Request processed in 78us (510us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764771.267097:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24111 00000100:00100000:3.0:1775764771.267450:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966944768:192.168.202.146@tcp:36:dbench.0 00000080:00200000:3.0:1775764771.267509:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.267513:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.267518:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:3.0:1775764771.267523:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.267526:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:3.0:1775764771.267527:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.267530:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:3.0:1775764771.267532:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.267534:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:3.0:1775764771.267535:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.267546:0:19723:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=COURSES.Y04, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), flags=1 00000080:00200000:3.0:1775764771.267549:0:19723:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=COURSES.Y04, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), intent=getattr 00010000:00010000:3.0:1775764771.272781:0:2381:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99c00/0x36f9bdc9e421c507 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x3/0x0 rrc: 10 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9e94 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.272799:0:2381:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99c00/0x36f9bdc9e421c507 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2c4:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9e94 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.272804:0:2381:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99c00/0x36f9bdc9e421c507 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2c4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9e94 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.272813:0:2381:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99c00/0x36f9bdc9e421c507 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2c4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9e94 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.272821:0:2381:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99c00/0x36f9bdc9e421c507 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2c4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9e94 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.272828:0:2381:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99c00/0x36f9bdc9e421c507 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2c4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9e94 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.272833:0:2381:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99c00/0x36f9bdc9e421c507 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2c4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9e94 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.272837:0:2381:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99c00/0x36f9bdc9e421c507 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2c4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9e94 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764771.272848:0:2381:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f99703d2680 x1862022966944512/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 3072/5384 e 0 to 0 dl 1775764787 ref 1 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:3.0:1775764771.272855:0:2381:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99c00/0x36f9bdc9e421c507 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9e94 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:3.0:1775764771.272867:0:2381:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c507 from it ffff8f994f0cdd88 00010000:00010000:3.0:1775764771.272869:0:2381:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99c00/0x36f9bdc9e421c507 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9e94 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.272875:0:2381:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99c00/0x36f9bdc9e421c507 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9e94 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.272895:0:2381:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98800/0x36f9bdc9e421c50e lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x3/0x0 rrc: 9 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9eb0 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.272900:0:2381:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98800/0x36f9bdc9e421c50e lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2c0:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9eb0 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.272904:0:2381:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98800/0x36f9bdc9e421c50e lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2c0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9eb0 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.272909:0:2381:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98800/0x36f9bdc9e421c50e lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2c0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9eb0 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.272912:0:2381:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98800/0x36f9bdc9e421c50e lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2c0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9eb0 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.272916:0:2381:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98800/0x36f9bdc9e421c50e lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2c0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9eb0 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.272920:0:2381:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98800/0x36f9bdc9e421c50e lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2c0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9eb0 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.272930:0:2381:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98800/0x36f9bdc9e421c50e lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2c0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9eb0 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764771.272947:0:2381:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f99703d2680 x1862022966944512/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 3072/5384 e 0 to 0 dl 1775764787 ref 2 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:3.0:1775764771.272954:0:2381:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98800/0x36f9bdc9e421c50e lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9eb0 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:3.0:1775764771.272962:0:2381:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c50e from it ffff8f994f0cf588 00010000:00010000:3.0:1775764771.272965:0:2381:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98800/0x36f9bdc9e421c50e lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9eb0 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.272972:0:2381:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98800/0x36f9bdc9e421c50e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9eb0 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.272984:0:2381:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98c00/0x36f9bdc9e421c515 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x3/0x0 rrc: 8 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9ecc expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.272991:0:2381:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98c00/0x36f9bdc9e421c515 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2c8:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9ecc expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.272995:0:2381:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98c00/0x36f9bdc9e421c515 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2c8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9ecc expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.273000:0:2381:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98c00/0x36f9bdc9e421c515 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2c8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9ecc expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.273008:0:2381:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98c00/0x36f9bdc9e421c515 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2c8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9ecc expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.273015:0:2381:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98c00/0x36f9bdc9e421c515 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2c8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9ecc expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.273022:0:2381:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98c00/0x36f9bdc9e421c515 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2c8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9ecc expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.273028:0:2381:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98c00/0x36f9bdc9e421c515 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2c8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9ecc expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764771.273036:0:2381:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f99703d2680 x1862022966944512/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 3072/5384 e 0 to 0 dl 1775764787 ref 3 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:3.0:1775764771.273043:0:2381:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98c00/0x36f9bdc9e421c515 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9ecc expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:3.0:1775764771.273052:0:2381:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c515 from it ffff8f994f0cd988 00010000:00010000:3.0:1775764771.273056:0:2381:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98c00/0x36f9bdc9e421c515 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9ecc expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.273063:0:2381:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98c00/0x36f9bdc9e421c515 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9ecc expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.273077:0:2381:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99a00/0x36f9bdc9e421c51c lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x3/0x0 rrc: 7 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9ee8 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.273088:0:2381:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99a00/0x36f9bdc9e421c51c lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2cc:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9ee8 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.273092:0:2381:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99a00/0x36f9bdc9e421c51c lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2cc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9ee8 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.273097:0:2381:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99a00/0x36f9bdc9e421c51c lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2cc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9ee8 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.273101:0:2381:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99a00/0x36f9bdc9e421c51c lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2cc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9ee8 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.273106:0:2381:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99a00/0x36f9bdc9e421c51c lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2cc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9ee8 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.273109:0:2381:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99a00/0x36f9bdc9e421c51c lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2cc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9ee8 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.273113:0:2381:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99a00/0x36f9bdc9e421c51c lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2cc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9ee8 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764771.273118:0:2381:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f99703d2680 x1862022966944512/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 3072/5384 e 0 to 0 dl 1775764787 ref 4 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:3.0:1775764771.273124:0:2381:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99a00/0x36f9bdc9e421c51c lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2cc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9ee8 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:3.0:1775764771.273137:0:2381:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c51c from it ffff8f994f0ccd88 00010000:00010000:3.0:1775764771.273140:0:2381:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99a00/0x36f9bdc9e421c51c lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2cc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9ee8 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.273146:0:2381:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99a00/0x36f9bdc9e421c51c lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2cc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9ee8 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.273159:0:2381:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98000/0x36f9bdc9e421c523 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x3/0x0 rrc: 6 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9f04 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.273168:0:2381:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98000/0x36f9bdc9e421c523 lrc: 4/1,0 mode: --/PR res: [0x200000401:0xec1:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9f04 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.273175:0:2381:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98000/0x36f9bdc9e421c523 lrc: 4/1,0 mode: --/PR res: [0x200000401:0xec1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9f04 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.273183:0:2381:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98000/0x36f9bdc9e421c523 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xec1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9f04 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.273190:0:2381:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98000/0x36f9bdc9e421c523 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xec1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9f04 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.273197:0:2381:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98000/0x36f9bdc9e421c523 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xec1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9f04 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.273201:0:2381:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98000/0x36f9bdc9e421c523 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xec1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9f04 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.273209:0:2381:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98000/0x36f9bdc9e421c523 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xec1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9f04 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764771.273214:0:2381:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f99703d2680 x1862022966944512/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 3072/5384 e 0 to 0 dl 1775764787 ref 5 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:3.0:1775764771.273218:0:2381:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98000/0x36f9bdc9e421c523 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xec1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9f04 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:3.0:1775764771.273224:0:2381:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c523 from it ffff8f994f0ce588 00010000:00010000:3.0:1775764771.273226:0:2381:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98000/0x36f9bdc9e421c523 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xec1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9f04 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.273230:0:2381:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98000/0x36f9bdc9e421c523 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xec1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9f04 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.273238:0:2381:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98400/0x36f9bdc9e421c52a lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x3/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9f20 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.273244:0:2381:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98400/0x36f9bdc9e421c52a lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2d6:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9f20 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.273247:0:2381:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98400/0x36f9bdc9e421c52a lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2d6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9f20 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.273252:0:2381:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98400/0x36f9bdc9e421c52a lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2d6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9f20 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.273259:0:2381:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98400/0x36f9bdc9e421c52a lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2d6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9f20 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.273263:0:2381:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98400/0x36f9bdc9e421c52a lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2d6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9f20 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.273267:0:2381:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98400/0x36f9bdc9e421c52a lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2d6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9f20 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.273270:0:2381:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98400/0x36f9bdc9e421c52a lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2d6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9f20 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764771.273274:0:2381:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f99703d2680 x1862022966944512/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 3072/5384 e 0 to 0 dl 1775764787 ref 6 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:3.0:1775764771.273278:0:2381:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98400/0x36f9bdc9e421c52a lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2d6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9f20 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:3.0:1775764771.273283:0:2381:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c52a from it ffff8f994f0cc188 00010000:00010000:3.0:1775764771.273285:0:2381:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98400/0x36f9bdc9e421c52a lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2d6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9f20 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.273288:0:2381:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98400/0x36f9bdc9e421c52a lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9f20 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.273295:0:2381:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98e00/0x36f9bdc9e421c531 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9f3c expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.273307:0:2381:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98e00/0x36f9bdc9e421c531 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2ca:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9f3c expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.273313:0:2381:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98e00/0x36f9bdc9e421c531 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2ca:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9f3c expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.273319:0:2381:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98e00/0x36f9bdc9e421c531 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2ca:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9f3c expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.273326:0:2381:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98e00/0x36f9bdc9e421c531 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2ca:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9f3c expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.273334:0:2381:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98e00/0x36f9bdc9e421c531 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2ca:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9f3c expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.273341:0:2381:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98e00/0x36f9bdc9e421c531 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2ca:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9f3c expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.273348:0:2381:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98e00/0x36f9bdc9e421c531 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2ca:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9f3c expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764771.273357:0:2381:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f99703d2680 x1862022966944512/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 3072/5384 e 0 to 0 dl 1775764787 ref 7 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:3.0:1775764771.273362:0:2381:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98e00/0x36f9bdc9e421c531 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2ca:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9f3c expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00010000:3.0:1775764771.273371:0:2381:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c531 from it ffff8f994f0cc588 00010000:00010000:3.0:1775764771.273373:0:2381:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98e00/0x36f9bdc9e421c531 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2ca:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9f3c expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.273388:0:2381:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98e00/0x36f9bdc9e421c531 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ca:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f9f3c expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000100:00100000:3.0:1775764771.273403:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966944512:192.168.202.146@tcp:63:ll_sa.0 00010000:00010000:3.0:1775764771.273437:0:19780:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99c00/0x36f9bdc9e421c507 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9e94 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764771.273511:0:19780:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x2c4:0x0](ffff8f9952e2c2c8): rc = 0 00000080:00200000:3.0:1775764771.273524:0:19780:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x2c4:0x0]: layout version change: 3 -> 3 00000080:00010000:3.0:1775764771.273530:0:19780:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2c4:0x0](ffff8f9952e2c2c8) for lock 0x36f9bdc9e421c507 00000080:00010000:3.0:1775764771.273541:0:19780:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c507 from it ffff8f994f0cdd88 00010000:00010000:3.0:1775764771.273543:0:19780:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99c00/0x36f9bdc9e421c507 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9e94 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.273548:0:19780:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99c00/0x36f9bdc9e421c507 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9e94 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764771.273564:0:19780:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98800/0x36f9bdc9e421c50e lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9eb0 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764771.273583:0:19780:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x2c0:0x0](ffff8f9952d8aa08): rc = 0 00000080:00200000:3.0:1775764771.273590:0:19780:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x2c0:0x0]: layout version change: 3 -> 3 00000080:00010000:3.0:1775764771.273595:0:19780:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2c0:0x0](ffff8f9952d8aa08) for lock 0x36f9bdc9e421c50e 00000080:00010000:3.0:1775764771.273597:0:19780:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c50e from it ffff8f994f0cf588 00010000:00010000:3.0:1775764771.273599:0:19780:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98800/0x36f9bdc9e421c50e lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9eb0 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.273604:0:19780:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98800/0x36f9bdc9e421c50e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9eb0 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764771.273612:0:19780:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98c00/0x36f9bdc9e421c515 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9ecc expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764771.273651:0:19780:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x2c8:0x0](ffff8f9952e29988): rc = 0 00000080:00200000:3.0:1775764771.273664:0:19780:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x2c8:0x0]: layout version change: 3 -> 3 00000080:00010000:3.0:1775764771.273672:0:19780:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2c8:0x0](ffff8f9952e29988) for lock 0x36f9bdc9e421c515 00000080:00010000:3.0:1775764771.273675:0:19780:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c515 from it ffff8f994f0cd988 00010000:00010000:3.0:1775764771.273677:0:19780:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98c00/0x36f9bdc9e421c515 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9ecc expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.273682:0:19780:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98c00/0x36f9bdc9e421c515 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9ecc expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764771.273692:0:19780:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99a00/0x36f9bdc9e421c51c lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2cc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9ee8 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764771.273711:0:19780:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x2cc:0x0](ffff8f9952e280c8): rc = 0 00000080:00200000:3.0:1775764771.273719:0:19780:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x2cc:0x0]: layout version change: 3 -> 3 00000080:00010000:3.0:1775764771.273723:0:19780:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2cc:0x0](ffff8f9952e280c8) for lock 0x36f9bdc9e421c51c 00000080:00010000:3.0:1775764771.273725:0:19780:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c51c from it ffff8f994f0ccd88 00010000:00010000:3.0:1775764771.273728:0:19780:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99a00/0x36f9bdc9e421c51c lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2cc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9ee8 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.273732:0:19780:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99a00/0x36f9bdc9e421c51c lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2cc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9ee8 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764771.273756:0:19780:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98000/0x36f9bdc9e421c523 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xec1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9f04 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764771.273775:0:19780:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0xec1:0x0](ffff8f9952eee3c8): rc = 0 00000080:00200000:3.0:1775764771.273782:0:19780:0:(file.c:6766:ll_layout_conf()) [0x200000401:0xec1:0x0]: layout version change: 3 -> 3 00000080:00010000:3.0:1775764771.273787:0:19780:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xec1:0x0](ffff8f9952eee3c8) for lock 0x36f9bdc9e421c523 00000080:00010000:3.0:1775764771.273789:0:19780:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c523 from it ffff8f994f0ce588 00010000:00010000:3.0:1775764771.273791:0:19780:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98000/0x36f9bdc9e421c523 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xec1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9f04 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.273795:0:19780:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98000/0x36f9bdc9e421c523 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xec1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9f04 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764771.273803:0:19780:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98400/0x36f9bdc9e421c52a lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2d6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9f20 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764771.273818:0:19780:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x2d6:0x0](ffff8f9952d96c08): rc = 0 00000080:00200000:3.0:1775764771.273825:0:19780:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x2d6:0x0]: layout version change: 3 -> 3 00000080:00010000:3.0:1775764771.273829:0:19780:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2d6:0x0](ffff8f9952d96c08) for lock 0x36f9bdc9e421c52a 00000080:00010000:3.0:1775764771.273832:0:19780:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c52a from it ffff8f994f0cc188 00010000:00010000:3.0:1775764771.273833:0:19780:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98400/0x36f9bdc9e421c52a lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2d6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9f20 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.273841:0:19780:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98400/0x36f9bdc9e421c52a lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9f20 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764771.273848:0:19780:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98e00/0x36f9bdc9e421c531 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2ca:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f9f3c expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764771.273862:0:19780:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x2ca:0x0](ffff8f9952e2f448): rc = 0 00000080:00200000:3.0:1775764771.273869:0:19780:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x2ca:0x0]: layout version change: 3 -> 3 00000080:00010000:3.0:1775764771.273873:0:19780:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2ca:0x0](ffff8f9952e2f448) for lock 0x36f9bdc9e421c531 00000080:00010000:3.0:1775764771.273875:0:19780:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c531 from it ffff8f994f0cc588 00010000:00010000:3.0:1775764771.273877:0:19780:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98e00/0x36f9bdc9e421c531 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2ca:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9f3c expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.273881:0:19780:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98e00/0x36f9bdc9e421c531 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ca:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9f3c expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764771.273906:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99c00/0x36f9bdc9e421c507 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9e94 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:3.0:1775764771.273916:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c507 from it ffffafbac1c7b900 00010000:00010000:3.0:1775764771.273918:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99c00/0x36f9bdc9e421c507 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9e94 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764771.273922:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99c00/0x36f9bdc9e421c507 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9e94 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:3.0:1775764771.273962:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file COURSES.Y04:[0x200000401:0x2c4:0x0](ffff8f9952e2c2c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764771.273969:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2c4:0x0](ffff8f9952e2c2c8),name=COURSES.Y04 00000002:00010000:3.0:1775764771.273984:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2c4:0x0]) in obj [0x200000401:0x2bb:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764771.273992:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970972e00/0x36f9bdc9e421bddd lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2c4:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f7e18 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.274003:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970972e00/0x36f9bdc9e421bddd lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2c4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f7e18 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764771.274010:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2bb:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764771.274015:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2c4:0x0](ffff8f9952e2c2c8) 00000080:00010000:3.0:1775764771.274018:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2c4:0x0](ffff8f9952e2c2c8) for lock 0x36f9bdc9e421bddd 00000080:00010000:3.0:1775764771.274021:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421bddd from it ffffafbac1c7bcb8 00010000:00010000:3.0:1775764771.274024:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970972e00/0x36f9bdc9e421bddd lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f7e18 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.274031:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970972e00/0x36f9bdc9e421bddd lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f7e18 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764771.274057:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2c4:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764771.274065:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2c4:0x0] 00000008:00010000:3.0:1775764771.274091:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1775764771.274094:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d5cb0, osc lock ffff8f9966ace4c0, flags 0x400001002 00010000:00010000:3.0:1775764771.274110:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000009c24ad6b/0x36f9bdc9e41cac6e lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x58:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eeb23 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 72 lru_type: 1 00010000:00010000:3.0:1775764771.274125:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000009c24ad6b/0x36f9bdc9e41cac6e lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x58:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eeb23 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 72 lru_type: 1 00000020:00200000:3.0:1775764771.274137:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1775764771.274140:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000009c24ad6b/0x36f9bdc9e41cac6e lrc: 6/0,1 mode: PW/PW res: [0x280000403:0x58:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4eeb23 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 72 lru_type: 1 00010000:00010000:3.0:1775764771.274147:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000009c24ad6b/0x36f9bdc9e41cac6e lrc: 5/0,0 mode: PW/PW res: [0x280000403:0x58:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eeb23 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 73 lru_type: 1 00000020:00200000:3.0:1775764771.274156:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1775764771.274165:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2c4:0x0] updating i_size 260096 i_blocks 515 00000020:00010000:3.0:1775764771.274169:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_release():245 00000020:00010000:3.0:1775764771.274172:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.274175:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d5cb0 (ffff8f9970314758/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.274179:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764771.274181:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d5cb0 (ffff8f9970314e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764771.274198:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2c4:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.274228:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.274233:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.274239:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:3.0:1775764771.274244:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.274247:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=16 00000080:00200000:3.0:1775764771.274250:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.274253:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:3.0:1775764771.274256:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.274259:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=16 00000080:00200000:3.0:1775764771.274261:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.274266:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=COURSES.Y04, flags=0 00000080:00200000:3.0:1775764771.274270:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 3 00000080:00200000:3.0:1775764771.274274:0:19723:0:(namei.c:2395:ll_unlink()) VFS Op:name=COURSES.Y04, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08) 00010000:00010000:3.0:1775764771.274305:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99c00/0x36f9bdc9e421c507 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c4:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f9e94 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764771.274325:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2c4:0x0]: losing layout lock 00010000:00010000:3.0:1775764771.274342:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970972e00/0x36f9bdc9e421bddd lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c4:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f7e18 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764771.274358:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2c4:0x0]: losing layout lock 00010000:00010000:3.0:1775764771.274383:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99c00/0x36f9bdc9e421c507 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2c4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f9e94 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764771.274388:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970972e00/0x36f9bdc9e421bddd lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2c4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f7e18 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764771.274391:0:19723:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1775764771.274394:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99c00/0x36f9bdc9e421c507 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2c4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f9e94 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764771.274404:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970972e00/0x36f9bdc9e421bddd lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2c4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f7e18 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000020:00100000:3.0:1775764771.274423:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1775764771.274430:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970682680 x1862022966945280/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764771.274457:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970682680 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966945280:192.168.202.146@tcp:36:dbench.0 00000100:00100000:3.0:1775764771.274519:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99707ef000 going to sleep for 11 seconds 00000100:00100000:3.0:1775764771.280832:0:19889:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970681c00 pname:cluuid:pid:xid:nid:opc:job ll_sa_19724:3f83546c-01a9-4156-8a97-44c446245d3e:19889:1862022966944384:192.168.202.146@tcp:37:ll_sa.0 00010000:00010000:3.0:1775764771.281345:0:19889:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5800/0x36f9bdc9e421c4f2 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x2/0x2 rrc: 4 type: IBT gid 0 flags: 0x10400400000000 nid: local remote: 0x2aac6a50e5f9e24 expref: -99 pid: 19889 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.281356:0:19889:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5800/0x36f9bdc9e421c4f2 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x2/0x2 rrc: 4 type: IBT gid 0 flags: 0x10400400000000 nid: local remote: 0x2aac6a50e5f9e24 expref: -99 pid: 19889 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.281371:0:19889:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00010000:00010000:3.0:1775764771.281429:0:19889:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421c569 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x292:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 19889 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.281437:0:19889:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421c569 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x292:0x0].0x0 bits 0x3/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19889 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.281460:0:19889:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4600/0x36f9bdc9e421c570 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x292:0x0].0x0 bits 0x0/0x0 rrc: 6 type: IBT flags: 0x10000000000000 pid: 19889 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.281464:0:19889:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4600/0x36f9bdc9e421c570 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x292:0x0].0x0 bits 0x3/0x0 rrc: 6 type: IBT flags: 0x0 pid: 19889 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.281473:0:19889:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c577 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x292:0x0].0x0 bits 0x0/0x0 rrc: 7 type: IBT flags: 0x10000000000000 pid: 19889 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.281477:0:19889:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c577 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x292:0x0].0x0 bits 0x3/0x0 rrc: 7 type: IBT flags: 0x0 pid: 19889 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.281488:0:19889:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c57e lrc: 3/1,0 mode: --/CR res: [0x200000401:0x292:0x0].0x0 bits 0x0/0x0 rrc: 8 type: IBT flags: 0x10000000000000 pid: 19889 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.281491:0:19889:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421c57e lrc: 3/1,0 mode: --/CR res: [0x200000401:0x292:0x0].0x0 bits 0x3/0x0 rrc: 8 type: IBT flags: 0x0 pid: 19889 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.281499:0:19889:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5c00/0x36f9bdc9e421c585 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x292:0x0].0x0 bits 0x0/0x0 rrc: 9 type: IBT flags: 0x10000000000000 pid: 19889 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.281502:0:19889:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5c00/0x36f9bdc9e421c585 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x292:0x0].0x0 bits 0x3/0x0 rrc: 9 type: IBT flags: 0x0 pid: 19889 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.281510:0:19889:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c58c lrc: 3/1,0 mode: --/CR res: [0x200000401:0x292:0x0].0x0 bits 0x0/0x0 rrc: 10 type: IBT flags: 0x10000000000000 pid: 19889 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.281514:0:19889:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c58c lrc: 3/1,0 mode: --/CR res: [0x200000401:0x292:0x0].0x0 bits 0x3/0x0 rrc: 10 type: IBT flags: 0x0 pid: 19889 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.281522:0:19889:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c593 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x292:0x0].0x0 bits 0x0/0x0 rrc: 11 type: IBT flags: 0x10000000000000 pid: 19889 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.281526:0:19889:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c593 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x292:0x0].0x0 bits 0x3/0x0 rrc: 11 type: IBT flags: 0x0 pid: 19889 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.281533:0:19889:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c59a lrc: 3/1,0 mode: --/CR res: [0x200000401:0x292:0x0].0x0 bits 0x0/0x0 rrc: 12 type: IBT flags: 0x10000000000000 pid: 19889 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.281537:0:19889:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c59a lrc: 3/1,0 mode: --/CR res: [0x200000401:0x292:0x0].0x0 bits 0x3/0x0 rrc: 12 type: IBT flags: 0x0 pid: 19889 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764771.281576:0:2382:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1775764771.281589:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970681c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966945536:192.168.202.146@tcp:63:ll_sa.0 00010000:00010000:0.0:1775764771.281788:0:11210:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1775764771.281795:0:11210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5800/0x36f9bdc9e421c4f2 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x2/0x2 rrc: 12 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x2aac6a50e5f9e24 expref: -99 pid: 19889 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.281800:0:11210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99435d5800 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:0.0:1775764771.281804:0:11210:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5800/0x36f9bdc9e421c4f2 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x2/0x2 rrc: 12 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x2aac6a50e5f9e24 expref: -99 pid: 19889 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.281809:0:11210:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5800/0x36f9bdc9e421c4f2 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x2/0x2 rrc: 12 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x2aac6a50e5f9e24 expref: -99 pid: 19889 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.281814:0:11210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5800/0x36f9bdc9e421c4f2 lrc: 3/0,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x2/0x2 rrc: 12 type: IBT gid 0 flags: 0x408400000000 nid: local remote: 0x2aac6a50e5f9e24 expref: -99 pid: 19889 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.281873:0:11210:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5800/0x36f9bdc9e421c4f2 lrc: 2/0,0 mode: --/PR res: [0x200000401:0x292:0x0].0x0 bits 0x2/0x2 rrc: 12 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x2aac6a50e5f9e24 expref: -99 pid: 19889 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.281877:0:11210:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764771.281897:0:11210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5800/0x36f9bdc9e421c4f2 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x292:0x0].0x0 bits 0x2/0x2 rrc: 12 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x2aac6a50e5f9e24 expref: -99 pid: 19889 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.281902:0:11210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5800/0x36f9bdc9e421c4f2 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x292:0x0].0x0 bits 0x2/0x2 rrc: 12 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x2aac6a50e5f9e24 expref: -99 pid: 19889 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.281908:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764771.281910:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764771.281911:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764771.281916:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1775764771.281922:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966945664:192.168.202.146@tcp:103:ldlm_bl.0 00000080:00200000:1.0:1775764771.287021:0:19888:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2de:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764771.287031:0:19888:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2de:0x0] 00000008:00010000:1.0:1775764771.287058:0:19888:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:1.0:1775764771.287062:0:19888:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f1c7430, osc lock ffff8f995875db50, flags 0x400041012 00010000:00010000:1.0:1775764771.287079:0:19888:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000acca1f81/0x36f9bdc9e41caffc lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x65:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef3ab expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 85 lru_type: 1 00010000:00010000:1.0:1775764771.287094:0:19888:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000acca1f81/0x36f9bdc9e41caffc lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x65:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef3ab expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 85 lru_type: 1 00010000:00010000:1.0:1775764771.287100:0:19888:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000acca1f81/0x36f9bdc9e41caffc lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x65:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ef3ab expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 85 lru_type: 1 00010000:00010000:1.0:1775764771.287105:0:19888:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000acca1f81/0x36f9bdc9e41caffc lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x65:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef3ab expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 86 lru_type: 1 00000020:00010000:1.0:1775764771.287114:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_release():245 00000020:00010000:1.0:1775764771.287117:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.287119:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f1c7430 (ffff8f99705b8998/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.287122:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764771.287125:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f1c7430 (ffff8f99705b8638/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764771.287144:0:19888:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2de:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764771.287534:0:19888:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2e0:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764771.287540:0:19888:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2e0:0x0] 00000008:00010000:1.0:1775764771.287566:0:19888:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:1.0:1775764771.287569:0:19888:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f1c7430, osc lock ffff8f995875db50, flags 0x400041012 00010000:00010000:1.0:1775764771.287591:0:19888:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000e275e2c0/0x36f9bdc9e420ed6c lrc: 2/1,0 mode: PR/PR res: [0x280000403:0x66:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5cd375 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764771.287602:0:19888:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000e275e2c0/0x36f9bdc9e420ed6c lrc: 2/1,0 mode: PR/PR res: [0x280000403:0x66:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5cd375 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764771.287633:0:19888:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000e275e2c0/0x36f9bdc9e420ed6c lrc: 4/1,0 mode: PR/PR res: [0x280000403:0x66:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5cd375 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764771.287642:0:19888:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000e275e2c0/0x36f9bdc9e420ed6c lrc: 3/0,0 mode: PR/PR res: [0x280000403:0x66:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5cd375 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00000020:00010000:1.0:1775764771.287653:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_release():245 00000020:00010000:1.0:1775764771.287655:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.287674:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f1c7430 (ffff8f99705b8998/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.287677:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764771.287693:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f1c7430 (ffff8f99705b8638/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764771.287707:0:19888:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2e0:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764771.287728:0:19888:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2d2:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764771.287731:0:19888:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2d2:0x0] 00000008:00010000:1.0:1775764771.287770:0:19888:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:1.0:1775764771.287772:0:19888:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f1c7430, osc lock ffff8f995875db50, flags 0x400041012 00010000:00010000:1.0:1775764771.287780:0:19888:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000df037f52/0x36f9bdc9e41cae58 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x5f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eefbb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 67 lru_type: 1 00010000:00010000:1.0:1775764771.287787:0:19888:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000df037f52/0x36f9bdc9e41cae58 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x5f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eefbb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 67 lru_type: 1 00010000:00010000:1.0:1775764771.287800:0:19888:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000df037f52/0x36f9bdc9e41cae58 lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x5f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4eefbb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 67 lru_type: 1 00010000:00010000:1.0:1775764771.287805:0:19888:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000df037f52/0x36f9bdc9e41cae58 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x5f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eefbb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 68 lru_type: 1 00000020:00010000:1.0:1775764771.287811:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_release():245 00000020:00010000:1.0:1775764771.287812:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.287813:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f1c7430 (ffff8f99705b8998/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.287815:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764771.287816:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f1c7430 (ffff8f99705b8638/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764771.287834:0:19888:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2d2:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.288051:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.288073:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.288080:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:3.0:1775764771.288086:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.288090:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:3.0:1775764771.288092:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.288094:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:3.0:1775764771.288096:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.288098:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:3.0:1775764771.288099:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.288126:0:19723:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=COURSES.PX, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), flags=1 00000080:00200000:3.0:1775764771.288130:0:19723:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=COURSES.PX, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), intent=getattr 00010000:00010000:3.0:1775764771.288154:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98800/0x36f9bdc9e421c50e lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9eb0 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:3.0:1775764771.288178:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c50e from it ffffafbac1c7b900 00010000:00010000:3.0:1775764771.288182:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98800/0x36f9bdc9e421c50e lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9eb0 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764771.288189:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98800/0x36f9bdc9e421c50e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9eb0 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:3.0:1775764771.288222:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file COURSES.PX:[0x200000401:0x2c0:0x0](ffff8f9952d8aa08), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764771.288237:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2c0:0x0](ffff8f9952d8aa08),name=COURSES.PX 00000002:00010000:3.0:1775764771.288250:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2c0:0x0]) in obj [0x200000401:0x2bb:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764771.288256:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970972000/0x36f9bdc9e421bd89 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2c0:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f7d1c expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.288272:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970972000/0x36f9bdc9e421bd89 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2c0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f7d1c expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764771.288276:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2bb:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764771.288281:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2c0:0x0](ffff8f9952d8aa08) 00000080:00010000:3.0:1775764771.288284:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2c0:0x0](ffff8f9952d8aa08) for lock 0x36f9bdc9e421bd89 00000080:00010000:3.0:1775764771.288286:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421bd89 from it ffffafbac1c7bcb8 00010000:00010000:3.0:1775764771.288288:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970972000/0x36f9bdc9e421bd89 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f7d1c expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.288305:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970972000/0x36f9bdc9e421bd89 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f7d1c expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764771.288350:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2c0:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764771.288355:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2c0:0x0] 00000008:00010000:3.0:1775764771.288378:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1775764771.288380:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d57b0, osc lock ffff8f9966ace4c0, flags 0x400001002 00010000:00010000:3.0:1775764771.288409:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000ceb22e30/0x36f9bdc9e41cabd4 lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x56:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee9b0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 0 00010000:00010000:3.0:1775764771.288418:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000ceb22e30/0x36f9bdc9e41cabd4 lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x56:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee9b0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 0 00000020:00200000:3.0:1775764771.288448:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1775764771.288452:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000ceb22e30/0x36f9bdc9e41cabd4 lrc: 6/0,1 mode: PW/PW res: [0x280000403:0x56:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ee9b0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 0 00010000:00010000:3.0:1775764771.288469:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000ceb22e30/0x36f9bdc9e41cabd4 lrc: 5/0,0 mode: PW/PW res: [0x280000403:0x56:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee9b0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 0 00000020:00200000:3.0:1775764771.288475:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1775764771.288482:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2c0:0x0] updating i_size 260096 i_blocks 515 00000020:00010000:3.0:1775764771.288484:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_release():245 00000020:00010000:3.0:1775764771.288485:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.288487:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d57b0 (ffff8f9970314758/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.288489:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764771.288491:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d57b0 (ffff8f9970314e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764771.288510:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2c0:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.288522:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.288524:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.288526:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:3.0:1775764771.288528:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.288530:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=16 00000080:00200000:3.0:1775764771.288531:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.288533:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:3.0:1775764771.288534:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.288536:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=16 00000080:00200000:3.0:1775764771.288537:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.288540:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=COURSES.PX, flags=0 00000080:00200000:3.0:1775764771.288542:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 3 00000080:00200000:3.0:1775764771.288552:0:19723:0:(namei.c:2395:ll_unlink()) VFS Op:name=COURSES.PX, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08) 00010000:00010000:3.0:1775764771.289588:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98800/0x36f9bdc9e421c50e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c0:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f9eb0 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764771.289769:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2c0:0x0]: losing layout lock 00010000:00010000:3.0:1775764771.289801:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970972000/0x36f9bdc9e421bd89 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c0:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f7d1c expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764771.289818:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2c0:0x0]: losing layout lock 00010000:00010000:3.0:1775764771.289859:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98800/0x36f9bdc9e421c50e lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2c0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f9eb0 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764771.289865:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970972000/0x36f9bdc9e421bd89 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2c0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f7d1c expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764771.289868:0:19723:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1775764771.289877:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98800/0x36f9bdc9e421c50e lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2c0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f9eb0 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764771.289886:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970972000/0x36f9bdc9e421bd89 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2c0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f7d1c expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000020:00100000:3.0:1775764771.289911:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1775764771.289920:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970682300 x1862022966945920/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764771.289954:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970682300 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966945920:192.168.202.146@tcp:36:dbench.0 00000080:00200000:1.0:1775764771.291200:0:19888:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xed6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764771.291209:0:19888:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xed6:0x0] 00000008:00010000:1.0:1775764771.291227:0:19888:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:1.0:1775764771.291230:0:19888:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f1c7430, osc lock ffff8f995875db50, flags 0x400041012 00010000:00010000:1.0:1775764771.291245:0:19888:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b03c608/0x36f9bdc9e421c12c lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x65f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f87fe expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764771.291254:0:19888:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b03c608/0x36f9bdc9e421c12c lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f87fe expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764771.291260:0:19888:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b03c608/0x36f9bdc9e421c12c lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5f87fe expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764771.291264:0:19888:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b03c608/0x36f9bdc9e421c12c lrc: 4/0,0 mode: PW/PW res: [0x280000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f87fe expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00010000:1.0:1775764771.291272:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_release():245 00000020:00010000:1.0:1775764771.291274:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.291276:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f1c7430 (ffff8f99705b8998/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.291278:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764771.291279:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f1c7430 (ffff8f99705b8638/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764771.291293:0:19888:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xed6:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764771.291310:0:19888:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xecd:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764771.291314:0:19888:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xecd:0x0] 00000008:00010000:1.0:1775764771.291325:0:19888:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:1.0:1775764771.291326:0:19888:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f1c7430, osc lock ffff8f995875db50, flags 0x400041012 00010000:00010000:1.0:1775764771.291336:0:19888:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f2afaa21/0x36f9bdc9e421b17a lrc: 3/0,1 mode: PW/PW res: [0x240000403:0x65b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f5839 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 0 00010000:00010000:1.0:1775764771.291345:0:19888:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f2afaa21/0x36f9bdc9e421b17a lrc: 3/0,1 mode: PW/PW res: [0x240000403:0x65b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f5839 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 0 00010000:00010000:1.0:1775764771.291359:0:19888:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f2afaa21/0x36f9bdc9e421b17a lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x65b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5f5839 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 0 00010000:00010000:1.0:1775764771.291367:0:19888:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f2afaa21/0x36f9bdc9e421b17a lrc: 4/0,0 mode: PW/PW res: [0x240000403:0x65b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f5839 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 0 00000020:00010000:1.0:1775764771.291375:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_release():245 00000020:00010000:1.0:1775764771.291386:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.291389:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f1c7430 (ffff8f99705b8998/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.291391:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764771.291393:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f1c7430 (ffff8f99705b8638/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764771.291405:0:19888:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xecd:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764771.291422:0:19888:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2dc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764771.291426:0:19888:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2dc:0x0] 00000008:00010000:1.0:1775764771.291437:0:19888:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:1.0:1775764771.291439:0:19888:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f1c7430, osc lock ffff8f995875db50, flags 0x400041012 00010000:00010000:1.0:1775764771.291449:0:19888:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000b908cb64/0x36f9bdc9e41cafa8 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x64:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef303 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 64 lru_type: 1 00010000:00010000:1.0:1775764771.291457:0:19888:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000b908cb64/0x36f9bdc9e41cafa8 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x64:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef303 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 64 lru_type: 1 00010000:00010000:1.0:1775764771.291463:0:19888:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000b908cb64/0x36f9bdc9e41cafa8 lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x64:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ef303 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 64 lru_type: 1 00010000:00010000:1.0:1775764771.291472:0:19888:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000b908cb64/0x36f9bdc9e41cafa8 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x64:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef303 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 65 lru_type: 1 00000020:00010000:1.0:1775764771.291478:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_release():245 00000020:00010000:1.0:1775764771.291479:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.291481:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f1c7430 (ffff8f99705b8998/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.291482:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764771.291483:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f1c7430 (ffff8f99705b8638/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764771.291491:0:19888:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2dc:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764771.291502:0:19888:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2e2:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764771.291505:0:19888:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2e2:0x0] 00000008:00010000:1.0:1775764771.291511:0:19888:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:1.0:1775764771.291512:0:19888:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f1c7430, osc lock ffff8f995875db50, flags 0x400041012 00010000:00010000:1.0:1775764771.291518:0:19888:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000070c338a1/0x36f9bdc9e41cb09d lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x67:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef556 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 27 lru_type: 1 00010000:00010000:1.0:1775764771.291523:0:19888:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000070c338a1/0x36f9bdc9e41cb09d lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x67:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef556 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 27 lru_type: 1 00010000:00010000:1.0:1775764771.291528:0:19888:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000070c338a1/0x36f9bdc9e41cb09d lrc: 4/0,1 mode: PW/PW res: [0x240000403:0x67:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ef556 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 27 lru_type: 1 00010000:00010000:1.0:1775764771.291533:0:19888:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000070c338a1/0x36f9bdc9e41cb09d lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x67:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef556 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 28 lru_type: 1 00000020:00010000:1.0:1775764771.291541:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_release():245 00000020:00010000:1.0:1775764771.291542:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.291544:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f1c7430 (ffff8f99705b8998/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.291545:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764771.291547:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f1c7430 (ffff8f99705b8638/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764771.291552:0:19888:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2e2:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764771.291561:0:19888:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xece:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764771.291563:0:19888:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xece:0x0] 00000008:00010000:1.0:1775764771.291569:0:19888:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:1.0:1775764771.291570:0:19888:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f1c7430, osc lock ffff8f995875db50, flags 0x400041012 00010000:00010000:1.0:1775764771.291574:0:19888:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000fb0a8ec1/0x36f9bdc9e421b2df lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x65b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f5cb5 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764771.291579:0:19888:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000fb0a8ec1/0x36f9bdc9e421b2df lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x65b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f5cb5 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764771.291583:0:19888:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000fb0a8ec1/0x36f9bdc9e421b2df lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x65b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5f5cb5 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764771.291587:0:19888:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000fb0a8ec1/0x36f9bdc9e421b2df lrc: 4/0,0 mode: PW/PW res: [0x280000403:0x65b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f5cb5 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 0 00000020:00010000:1.0:1775764771.291591:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_release():245 00000020:00010000:1.0:1775764771.291593:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.291594:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f1c7430 (ffff8f99705b8998/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.291596:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764771.291597:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f1c7430 (ffff8f99705b8638/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764771.291602:0:19888:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xece:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:0.0:1775764771.291972:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966945664:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764771.292159:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970681180 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966945024:192.168.202.146@tcp:36:dbench.0 00000080:00200000:0.0:1775764771.292212:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x29d:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764771.292220:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1775764771.292231:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:0.0:1775764771.292232:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:0.0:1775764771.292253:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x29d:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764771.292578:0:19724:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x29d:0x0](ffff8f9952d8db88) 00000008:00010000:0.0:1775764771.292607:0:19724:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 1 [cma]time 1775764054 1775764054 0 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f01cb6c5/0x36f9bdc9e41ca6ef lrc: 1/0,0 mode: PW/PW res: [0x240000403:0x48:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4edcb3 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 42 lru_type: 1 00000100:00100000:3.0:1775764771.296680:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413e40 going to sleep for 11 seconds 00000080:00200000:0.0:1775764771.296839:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.296845:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.296850:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:0.0:1775764771.296855:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.296857:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:0.0:1775764771.296859:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.296861:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:0.0:1775764771.296863:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.296865:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=COREL, flags=17 00000080:00200000:0.0:1775764771.296867:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x292:0x0](ffff8f9952d89148), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.296876:0:19724:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=ROSE.CPT, dir=[0x200000401:0x292:0x0](ffff8f9952d89148), flags=1 00000080:00200000:0.0:1775764771.296879:0:19724:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=ROSE.CPT, dir=[0x200000401:0x292:0x0](ffff8f9952d89148), intent=getattr 00000080:00200000:1.0:1775764771.299367:0:19890:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2a5:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764771.299531:0:19890:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2a5:0x0] 00000008:00010000:1.0:1775764771.299555:0:19890:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:1.0:1775764771.299558:0:19890:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f1c7430, osc lock ffff8f995875db50, flags 0x400041012 00010000:00010000:1.0:1775764771.299572:0:19890:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000009eadeed7/0x36f9bdc9e41ca807 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x4c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4edf53 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 56 lru_type: 1 00010000:00010000:1.0:1775764771.299583:0:19890:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000009eadeed7/0x36f9bdc9e41ca807 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x4c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4edf53 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 56 lru_type: 1 00010000:00010000:1.0:1775764771.299588:0:19890:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000009eadeed7/0x36f9bdc9e41ca807 lrc: 4/0,1 mode: PW/PW res: [0x240000403:0x4c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4edf53 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 56 lru_type: 1 00010000:00010000:1.0:1775764771.299593:0:19890:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000009eadeed7/0x36f9bdc9e41ca807 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x4c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4edf53 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 57 lru_type: 1 00000020:00010000:1.0:1775764771.299601:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_release():245 00000020:00010000:1.0:1775764771.299603:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.299604:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f1c7430 (ffff8f99705b8998/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.299607:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764771.299735:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f1c7430 (ffff8f99705b8638/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764771.299800:0:19890:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2a5:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764771.299816:0:19890:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2a3:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764771.299819:0:19890:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2a3:0x0] 00000008:00010000:1.0:1775764771.299828:0:19890:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:1.0:1775764771.299829:0:19890:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f1c7430, osc lock ffff8f995875db50, flags 0x400041012 00010000:00010000:1.0:1775764771.299837:0:19890:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000609c9638/0x36f9bdc9e41ca7ba lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x4b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ede9d expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 56 lru_type: 1 00010000:00010000:1.0:1775764771.299843:0:19890:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000609c9638/0x36f9bdc9e41ca7ba lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x4b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ede9d expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 56 lru_type: 1 00010000:00010000:1.0:1775764771.299848:0:19890:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000609c9638/0x36f9bdc9e41ca7ba lrc: 4/0,1 mode: PW/PW res: [0x240000403:0x4b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ede9d expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 56 lru_type: 1 00010000:00010000:1.0:1775764771.299853:0:19890:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000609c9638/0x36f9bdc9e41ca7ba lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x4b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ede9d expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 57 lru_type: 1 00000020:00010000:1.0:1775764771.299858:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_release():245 00000020:00010000:1.0:1775764771.299860:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.299861:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f1c7430 (ffff8f99705b8998/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.299862:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764771.299864:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f1c7430 (ffff8f99705b8638/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764771.299869:0:19890:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2a3:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764771.299879:0:19890:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2a7:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764771.299881:0:19890:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2a7:0x0] 00000008:00010000:1.0:1775764771.299887:0:19890:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:1.0:1775764771.299888:0:19890:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f1c7430, osc lock ffff8f995875db50, flags 0x400041012 00010000:00010000:1.0:1775764771.299893:0:19890:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c6ec0b7f/0x36f9bdc9e41ca84d lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x4d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4edffb expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 69 lru_type: 1 00010000:00010000:1.0:1775764771.299898:0:19890:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c6ec0b7f/0x36f9bdc9e41ca84d lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x4d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4edffb expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 69 lru_type: 1 00010000:00010000:1.0:1775764771.299903:0:19890:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c6ec0b7f/0x36f9bdc9e41ca84d lrc: 4/0,1 mode: PW/PW res: [0x240000403:0x4d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4edffb expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 69 lru_type: 1 00010000:00010000:1.0:1775764771.299907:0:19890:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c6ec0b7f/0x36f9bdc9e41ca84d lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x4d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4edffb expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 70 lru_type: 1 00000020:00010000:1.0:1775764771.299912:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_release():245 00000020:00010000:1.0:1775764771.299913:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.299914:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f1c7430 (ffff8f99705b8998/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.299915:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764771.299916:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f1c7430 (ffff8f99705b8638/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764771.299922:0:19890:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2a7:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764771.299930:0:19890:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x295:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764771.299933:0:19890:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x295:0x0] 00000008:00010000:1.0:1775764771.299998:0:19890:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:1.0:1775764771.300000:0:19890:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f1c7430, osc lock ffff8f995875db50, flags 0x400041012 00010000:00010000:1.0:1775764771.300007:0:19890:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001cab28fe/0x36f9bdc9e41ca5de lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x44:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eda36 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 91 lru_type: 1 00010000:00010000:1.0:1775764771.300053:0:19890:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001cab28fe/0x36f9bdc9e41ca5de lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x44:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eda36 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 91 lru_type: 1 00010000:00010000:1.0:1775764771.300061:0:19890:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001cab28fe/0x36f9bdc9e41ca5de lrc: 4/0,1 mode: PW/PW res: [0x240000403:0x44:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4eda36 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 91 lru_type: 1 00010000:00010000:1.0:1775764771.300075:0:19890:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001cab28fe/0x36f9bdc9e41ca5de lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x44:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eda36 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 92 lru_type: 1 00000020:00010000:1.0:1775764771.300085:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_release():245 00000020:00010000:1.0:1775764771.300086:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.300088:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f1c7430 (ffff8f99705b8998/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.300090:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764771.300091:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f1c7430 (ffff8f99705b8638/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764771.300100:0:19890:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x295:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764771.300112:0:19890:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2a1:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764771.300115:0:19890:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2a1:0x0] 00000008:00010000:1.0:1775764771.300123:0:19890:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:1.0:1775764771.300125:0:19890:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f1c7430, osc lock ffff8f995875db50, flags 0x400041012 00010000:00010000:1.0:1775764771.300133:0:19890:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000b02e1101/0x36f9bdc9e41ca774 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x4a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eddf5 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 69 lru_type: 1 00010000:00010000:1.0:1775764771.300141:0:19890:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000b02e1101/0x36f9bdc9e41ca774 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x4a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eddf5 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 69 lru_type: 1 00010000:00010000:1.0:1775764771.300148:0:19890:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000b02e1101/0x36f9bdc9e41ca774 lrc: 4/0,1 mode: PW/PW res: [0x240000403:0x4a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4eddf5 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 69 lru_type: 1 00010000:00010000:1.0:1775764771.300155:0:19890:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000b02e1101/0x36f9bdc9e41ca774 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x4a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eddf5 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 70 lru_type: 1 00000020:00010000:1.0:1775764771.300169:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_release():245 00000020:00010000:1.0:1775764771.300172:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.300174:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f1c7430 (ffff8f99705b8998/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.300177:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764771.300179:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f1c7430 (ffff8f99705b8638/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764771.300190:0:19890:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2a1:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764771.300203:0:19890:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x29a:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764771.300206:0:19890:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x29a:0x0] 00000008:00010000:1.0:1775764771.300213:0:19890:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:1.0:1775764771.300215:0:19890:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f1c7430, osc lock ffff8f995875db50, flags 0x400041012 00010000:00010000:1.0:1775764771.300222:0:19890:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000054dab604/0x36f9bdc9e41ca678 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x46:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4edb94 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764771.300228:0:19890:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000054dab604/0x36f9bdc9e41ca678 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x46:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4edb94 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764771.300233:0:19890:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000054dab604/0x36f9bdc9e41ca678 lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x46:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4edb94 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764771.300238:0:19890:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000054dab604/0x36f9bdc9e41ca678 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x46:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4edb94 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00000020:00010000:1.0:1775764771.300244:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_release():245 00000020:00010000:1.0:1775764771.300245:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.300246:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f1c7430 (ffff8f99705b8998/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.300248:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764771.300250:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f1c7430 (ffff8f99705b8638/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764771.300256:0:19890:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x29a:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764771.300265:0:19890:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x29c:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764771.300267:0:19890:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x29c:0x0] 00000008:00010000:1.0:1775764771.300273:0:19890:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:1.0:1775764771.300274:0:19890:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f1c7430, osc lock ffff8f995875db50, flags 0x400041012 00010000:00010000:1.0:1775764771.300279:0:19890:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000002f218f80/0x36f9bdc9e41ca6be lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x47:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4edc3c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764771.300284:0:19890:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000002f218f80/0x36f9bdc9e41ca6be lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x47:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4edc3c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764771.300288:0:19890:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000002f218f80/0x36f9bdc9e41ca6be lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x47:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4edc3c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764771.300292:0:19890:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000002f218f80/0x36f9bdc9e41ca6be lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x47:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4edc3c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00000020:00010000:1.0:1775764771.300297:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_release():245 00000020:00010000:1.0:1775764771.300299:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.300301:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f1c7430 (ffff8f99705b8998/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.300303:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764771.300305:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f1c7430 (ffff8f99705b8638/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764771.300314:0:19890:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x29c:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764771.300330:0:19890:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x294:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764771.300334:0:19890:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x294:0x0] 00000008:00010000:1.0:1775764771.300345:0:19890:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:1.0:1775764771.300347:0:19890:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f1c7430, osc lock ffff8f995875db50, flags 0x400041012 00010000:00010000:1.0:1775764771.300355:0:19890:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000031ed7a52/0x36f9bdc9e41ca591 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x43:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ed95d expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 91 lru_type: 1 00010000:00010000:1.0:1775764771.300363:0:19890:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000031ed7a52/0x36f9bdc9e41ca591 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x43:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ed95d expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 91 lru_type: 1 00010000:00010000:1.0:1775764771.300368:0:19890:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000031ed7a52/0x36f9bdc9e41ca591 lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x43:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ed95d expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 91 lru_type: 1 00010000:00010000:1.0:1775764771.300373:0:19890:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000031ed7a52/0x36f9bdc9e41ca591 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x43:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ed95d expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 92 lru_type: 1 00000020:00010000:1.0:1775764771.300378:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_release():245 00000020:00010000:1.0:1775764771.300379:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.300486:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f1c7430 (ffff8f99705b8998/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.300488:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764771.300489:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f1c7430 (ffff8f99705b8638/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764771.300498:0:19890:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x294:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:1.0:1775764771.300721:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421c569 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fa046 expref: -99 pid: 19889 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:1.0:1775764771.300733:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c569 from it ffffafbac1c5b900 00010000:00010000:1.0:1775764771.300735:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421c569 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fa046 expref: -99 pid: 19889 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764771.300805:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421c569 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fa046 expref: -99 pid: 19889 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:1.0:1775764771.300892:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file ROSE.CPT:[0x200000401:0x2a5:0x0](ffff8f9952d13a88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764771.300897:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2a5:0x0](ffff8f9952d13a88),name=ROSE.CPT 00000002:00010000:1.0:1775764771.300912:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2a5:0x0]) in obj [0x200000401:0x292:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764771.300925:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970ade600/0x36f9bdc9e4217735 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2a5:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e92db expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764771.300931:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970ade600/0x36f9bdc9e4217735 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2a5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e92db expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764771.300936:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x292:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764771.300956:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2a5:0x0](ffff8f9952d13a88) 00000080:00010000:1.0:1775764771.300959:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2a5:0x0](ffff8f9952d13a88) for lock 0x36f9bdc9e4217735 00000080:00010000:1.0:1775764771.300961:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e4217735 from it ffffafbac1c5bcb8 00010000:00010000:1.0:1775764771.300964:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970ade600/0x36f9bdc9e4217735 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5e92db expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764771.300968:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970ade600/0x36f9bdc9e4217735 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e92db expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764771.300994:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2a5:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764771.300997:0:19724:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2a5:0x0] 00000008:00010000:1.0:1775764771.301023:0:19724:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764771.301025:0:19724:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f1c7430, osc lock ffff8f995875db50, flags 0x400001002 00010000:00010000:1.0:1775764771.301036:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000009eadeed7/0x36f9bdc9e41ca807 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x4c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4edf53 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 57 lru_type: 1 00010000:00010000:1.0:1775764771.301057:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000009eadeed7/0x36f9bdc9e41ca807 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x4c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4edf53 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 57 lru_type: 1 00000020:00200000:1.0:1775764771.301069:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764771.301087:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000009eadeed7/0x36f9bdc9e41ca807 lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x4c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4edf53 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 57 lru_type: 1 00010000:00010000:1.0:1775764771.301093:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000009eadeed7/0x36f9bdc9e41ca807 lrc: 4/0,0 mode: PW/PW res: [0x240000403:0x4c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4edf53 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 58 lru_type: 1 00000020:00200000:1.0:1775764771.301099:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764771.301106:0:19724:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2a5:0x0] updating i_size 260096 i_blocks 1 00000020:00010000:1.0:1775764771.301108:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_release():245 00000020:00010000:1.0:1775764771.301110:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.301119:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f1c7430 (ffff8f99705b8998/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.301122:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764771.301125:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f1c7430 (ffff8f99705b8638/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764771.301139:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2a5:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764771.301198:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.301203:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.301208:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:1.0:1775764771.301212:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.301214:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=16 00000080:00200000:1.0:1775764771.301216:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764771.301218:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:1.0:1775764771.301220:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764771.301222:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=COREL, flags=16 00000080:00200000:1.0:1775764771.301235:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x292:0x0](ffff8f9952d89148), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764771.301239:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=ROSE.CPT, flags=0 00000080:00200000:1.0:1775764771.301242:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x292:0x0](ffff8f9952d89148), inode mode 41c0 mask 3 00000080:00200000:1.0:1775764771.301255:0:19724:0:(namei.c:2395:ll_unlink()) VFS Op:name=ROSE.CPT, dir=[0x200000401:0x292:0x0](ffff8f9952d89148) 00010000:00010000:1.0:1775764771.301285:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421c569 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a5:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5fa046 expref: -99 pid: 19889 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764771.301301:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2a5:0x0]: losing layout lock 00010000:00010000:1.0:1775764771.301320:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970ade600/0x36f9bdc9e4217735 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a5:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5e92db expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764771.301335:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2a5:0x0]: losing layout lock 00010000:00010000:1.0:1775764771.301369:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421c569 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2a5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fa046 expref: -99 pid: 19889 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764771.301375:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970ade600/0x36f9bdc9e4217735 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2a5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5e92db expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764771.301509:0:19724:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1775764771.301512:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5a00/0x36f9bdc9e421c569 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2a5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fa046 expref: -99 pid: 19889 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764771.301522:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970ade600/0x36f9bdc9e4217735 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2a5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5e92db expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000020:00100000:1.0:1775764771.301538:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 2 is allocated opc 36, max 7 00000100:00100000:1.0:1775764771.301547:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970b51c00 x1862022966946304/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764771.301566:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970b51c00 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966946304:192.168.202.146@tcp:36:dbench.0 00010000:00010000:3.0:1775764771.301690:0:19889:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c5c4 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x292:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 19889 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.301698:0:19889:0:(ldlm_request.c:1275:ldlm_cli_lock_create_pack()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c5c4 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x292:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19889 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764771.304567:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f997130d600 going to sleep for 11 seconds 00000080:00200000:1.0:1775764771.306676:0:19890:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x298:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764771.306686:0:19890:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x298:0x0] 00000008:00010000:1.0:1775764771.306714:0:19890:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:1.0:1775764771.306718:0:19890:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f1c7430, osc lock ffff8f995875db50, flags 0x400041012 00010000:00010000:1.0:1775764771.306863:0:19890:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000139dd483/0x36f9bdc9e4210797 lrc: 2/1,0 mode: PR/PR res: [0x280000403:0x45:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5d270c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764771.306876:0:19890:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000139dd483/0x36f9bdc9e4210797 lrc: 2/1,0 mode: PR/PR res: [0x280000403:0x45:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5d270c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764771.306883:0:19890:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000139dd483/0x36f9bdc9e4210797 lrc: 4/1,0 mode: PR/PR res: [0x280000403:0x45:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5d270c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764771.306888:0:19890:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000139dd483/0x36f9bdc9e4210797 lrc: 3/0,0 mode: PR/PR res: [0x280000403:0x45:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5d270c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00000020:00010000:1.0:1775764771.306897:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_release():245 00000020:00010000:1.0:1775764771.306899:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.306901:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f1c7430 (ffff8f99705b8998/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.306903:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764771.306905:0:19890:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f1c7430 (ffff8f99705b8638/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764771.306922:0:19890:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x298:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00010000:3.0:1775764771.309085:0:2382:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c5af from it ffff8f994f0cc588 00010000:00010000:3.0:1775764771.309091:0:2382:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebe00/0x36f9bdc9e421c5af lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2d4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa17a expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.309098:0:2382:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebe00/0x36f9bdc9e421c5af lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa17a expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.309116:0:2382:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c5b6 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x3/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa196 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.309125:0:2382:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c5b6 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2d0:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa196 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.309131:0:2382:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c5b6 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2d0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa196 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.309138:0:2382:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c5b6 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2d0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa196 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.309146:0:2382:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c5b6 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2d0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa196 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.309154:0:2382:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c5b6 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2d0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa196 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.309161:0:2382:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c5b6 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2d0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa196 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.309173:0:2382:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c5b6 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2d0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa196 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764771.309182:0:2382:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f99703d2d80 x1862022966946048/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 2272/3912 e 0 to 0 dl 1775764787 ref 3 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:3.0:1775764771.309190:0:2382:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c5b6 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2d0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa196 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.309205:0:19780:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebe00/0x36f9bdc9e421c5af lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2d4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa17a expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764771.309254:0:19780:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x2d4:0x0](ffff8f9952d95b88): rc = 0 00000080:00200000:3.0:1775764771.309266:0:19780:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x2d4:0x0]: layout version change: 3 -> 3 00000080:00010000:3.0:1775764771.309272:0:19780:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2d4:0x0](ffff8f9952d95b88) for lock 0x36f9bdc9e421c5af 00000080:00010000:3.0:1775764771.309275:0:19780:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c5af from it ffff8f994f0cc588 00010000:00010000:3.0:1775764771.309277:0:19780:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebe00/0x36f9bdc9e421c5af lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2d4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fa17a expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.309282:0:19780:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebe00/0x36f9bdc9e421c5af lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fa17a expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:3.0:1775764771.309296:0:2382:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c5b6 from it ffff8f994f0ced88 00010000:00010000:3.0:1775764771.309299:0:2382:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c5b6 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2d0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa196 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.309306:0:2382:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c5b6 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa196 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.309320:0:2382:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb000/0x36f9bdc9e421c5bd lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x3/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa1b2 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.309341:0:2382:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb000/0x36f9bdc9e421c5bd lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2da:0x0].0x0 bits 0x3/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa1b2 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.309349:0:2382:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb000/0x36f9bdc9e421c5bd lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2da:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa1b2 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.309356:0:2382:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb000/0x36f9bdc9e421c5bd lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2da:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa1b2 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.309362:0:2382:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb000/0x36f9bdc9e421c5bd lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2da:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa1b2 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.309367:0:2382:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb000/0x36f9bdc9e421c5bd lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2da:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa1b2 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.309372:0:2382:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb000/0x36f9bdc9e421c5bd lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2da:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa1b2 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.309376:0:2382:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb000/0x36f9bdc9e421c5bd lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2da:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa1b2 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764771.309381:0:2382:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f99703d2d80 x1862022966946048/t0(0) o63->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:24/4 lens 2272/3912 e 0 to 0 dl 1775764787 ref 3 fl Interpret:RQU/200/0 rc 0/0 job:'ll_sa.0' uid:0 gid:0 projid:4294967295 00000002:00010000:3.0:1775764771.309389:0:2382:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb000/0x36f9bdc9e421c5bd lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2da:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa1b2 expref: -99 pid: 19887 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.309401:0:19780:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c5b6 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2d0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa196 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764771.309433:0:19780:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x2d0:0x0](ffff8f9952d94b08): rc = 0 00000080:00200000:3.0:1775764771.309448:0:19780:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x2d0:0x0]: layout version change: 3 -> 3 00000080:00010000:3.0:1775764771.309456:0:19780:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2d0:0x0](ffff8f9952d94b08) for lock 0x36f9bdc9e421c5b6 00000080:00010000:3.0:1775764771.309459:0:19780:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c5b6 from it ffff8f994f0ced88 00010000:00010000:3.0:1775764771.309461:0:19780:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c5b6 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2d0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fa196 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.309466:0:19780:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c5b6 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fa196 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:3.0:1775764771.309478:0:2382:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c5bd from it ffff8f994f0cf988 00010000:00010000:3.0:1775764771.309480:0:2382:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb000/0x36f9bdc9e421c5bd lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2da:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa1b2 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.309485:0:2382:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb000/0x36f9bdc9e421c5bd lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2da:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa1b2 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000100:00100000:3.0:1775764771.309496:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966946048:192.168.202.146@tcp:63:ll_sa.0 00010000:00010000:3.0:1775764771.309509:0:19780:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb000/0x36f9bdc9e421c5bd lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2da:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa1b2 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764771.309534:0:19780:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x2da:0x0](ffff8f9952d95348): rc = 0 00000080:00200000:3.0:1775764771.309542:0:19780:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x2da:0x0]: layout version change: 3 -> 3 00000080:00010000:3.0:1775764771.309547:0:19780:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2da:0x0](ffff8f9952d95348) for lock 0x36f9bdc9e421c5bd 00000080:00010000:3.0:1775764771.309549:0:19780:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c5bd from it ffff8f994f0cf988 00010000:00010000:3.0:1775764771.309551:0:19780:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb000/0x36f9bdc9e421c5bd lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2da:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fa1b2 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.309555:0:19780:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb000/0x36f9bdc9e421c5bd lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2da:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fa1b2 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764771.310516:0:19888:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2ce:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764771.310531:0:19888:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2ce:0x0] 00000008:00010000:1.0:1775764771.310555:0:19888:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:1.0:1775764771.310558:0:19888:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f1c7430, osc lock ffff8f995875db50, flags 0x400041012 00010000:00010000:1.0:1775764771.310574:0:19888:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f970e87b/0x36f9bdc9e41cadcc lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x5d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eee6b expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 37 lru_type: 1 00010000:00010000:1.0:1775764771.310585:0:19888:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f970e87b/0x36f9bdc9e41cadcc lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x5d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eee6b expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 37 lru_type: 1 00010000:00010000:1.0:1775764771.310592:0:19888:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f970e87b/0x36f9bdc9e41cadcc lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x5d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4eee6b expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 37 lru_type: 1 00010000:00010000:1.0:1775764771.310597:0:19888:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f970e87b/0x36f9bdc9e41cadcc lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x5d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eee6b expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 38 lru_type: 1 00000020:00010000:1.0:1775764771.310606:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_release():245 00000020:00010000:1.0:1775764771.310608:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.310610:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f1c7430 (ffff8f99705b8998/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.310640:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764771.310643:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f1c7430 (ffff8f99705b8638/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764771.310671:0:19888:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2ce:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764771.310689:0:19888:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2d4:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764771.310692:0:19888:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2d4:0x0] 00000008:00010000:1.0:1775764771.310701:0:19888:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:1.0:1775764771.310702:0:19888:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f1c7430, osc lock ffff8f995875db50, flags 0x400041012 00010000:00010000:1.0:1775764771.310709:0:19888:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f444aa51/0x36f9bdc9e41cae97 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x60:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef055 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 137 lru_type: 1 00010000:00010000:1.0:1775764771.310715:0:19888:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f444aa51/0x36f9bdc9e41cae97 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x60:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef055 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 137 lru_type: 1 00010000:00010000:1.0:1775764771.310721:0:19888:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f444aa51/0x36f9bdc9e41cae97 lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x60:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ef055 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 137 lru_type: 1 00010000:00010000:1.0:1775764771.310726:0:19888:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f444aa51/0x36f9bdc9e41cae97 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x60:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef055 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 138 lru_type: 1 00000020:00010000:1.0:1775764771.310731:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_release():245 00000020:00010000:1.0:1775764771.310732:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.310733:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f1c7430 (ffff8f99705b8998/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.310735:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764771.310755:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f1c7430 (ffff8f99705b8638/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764771.310763:0:19888:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2d4:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764771.310774:0:19888:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2d0:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764771.310776:0:19888:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2d0:0x0] 00000008:00010000:1.0:1775764771.310783:0:19888:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:1.0:1775764771.310784:0:19888:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f1c7430, osc lock ffff8f995875db50, flags 0x400041012 00010000:00010000:1.0:1775764771.310790:0:19888:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c8469d80/0x36f9bdc9e41cae0b lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x5e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eef05 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 37 lru_type: 1 00010000:00010000:1.0:1775764771.310795:0:19888:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c8469d80/0x36f9bdc9e41cae0b lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x5e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eef05 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 37 lru_type: 1 00010000:00010000:1.0:1775764771.310829:0:19888:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c8469d80/0x36f9bdc9e41cae0b lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x5e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4eef05 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 37 lru_type: 1 00010000:00010000:1.0:1775764771.310837:0:19888:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c8469d80/0x36f9bdc9e41cae0b lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x5e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eef05 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 38 lru_type: 1 00000020:00010000:1.0:1775764771.310851:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_release():245 00000020:00010000:1.0:1775764771.310852:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.310854:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f1c7430 (ffff8f99705b8998/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.310856:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764771.310857:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f1c7430 (ffff8f99705b8638/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764771.310864:0:19888:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2d0:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764771.310881:0:19888:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2da:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764771.310884:0:19888:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2da:0x0] 00000008:00010000:1.0:1775764771.310890:0:19888:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:1.0:1775764771.310892:0:19888:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f1c7430, osc lock ffff8f995875db50, flags 0x400041012 00010000:00010000:1.0:1775764771.310896:0:19888:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000000b928e73/0x36f9bdc9e41caf70 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x63:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef24d expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 64 lru_type: 1 00010000:00010000:1.0:1775764771.310902:0:19888:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000000b928e73/0x36f9bdc9e41caf70 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x63:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef24d expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 64 lru_type: 1 00010000:00010000:1.0:1775764771.310919:0:19888:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000000b928e73/0x36f9bdc9e41caf70 lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x63:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ef24d expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 64 lru_type: 1 00010000:00010000:1.0:1775764771.310924:0:19888:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000000b928e73/0x36f9bdc9e41caf70 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x63:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef24d expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 65 lru_type: 1 00000100:00100000:3.0:1775764771.311327:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970682300 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966945920:192.168.202.146@tcp:36:dbench.0 00000080:00200000:3.0:1775764771.311385:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.311389:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.311394:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:3.0:1775764771.311400:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.311404:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:3.0:1775764771.311406:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.311409:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:3.0:1775764771.311412:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.311414:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:3.0:1775764771.311417:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.311430:0:19723:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=STUDENTS.PX, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), flags=1 00000080:00200000:3.0:1775764771.311435:0:19723:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=STUDENTS.PX, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), intent=getattr 00010000:00010000:3.0:1775764771.311447:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98c00/0x36f9bdc9e421c515 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9ecc expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:3.0:1775764771.311463:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c515 from it ffffafbac1c7b900 00010000:00010000:3.0:1775764771.311465:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98c00/0x36f9bdc9e421c515 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9ecc expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764771.311469:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98c00/0x36f9bdc9e421c515 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9ecc expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:3.0:1775764771.311488:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file STUDENTS.PX:[0x200000401:0x2c8:0x0](ffff8f9952e29988), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764771.311492:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2c8:0x0](ffff8f9952e29988),name=STUDENTS.PX 00000002:00010000:3.0:1775764771.311504:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2c8:0x0]) in obj [0x200000401:0x2bb:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764771.311510:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970972600/0x36f9bdc9e421be54 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2c8:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f7f84 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.311515:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970972600/0x36f9bdc9e421be54 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2c8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f7f84 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764771.311520:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2bb:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764771.311523:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2c8:0x0](ffff8f9952e29988) 00000080:00010000:3.0:1775764771.311526:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2c8:0x0](ffff8f9952e29988) for lock 0x36f9bdc9e421be54 00000080:00010000:3.0:1775764771.311528:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421be54 from it ffffafbac1c7bcb8 00010000:00010000:3.0:1775764771.311530:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970972600/0x36f9bdc9e421be54 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2c8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f7f84 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.311534:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970972600/0x36f9bdc9e421be54 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f7f84 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764771.311558:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2c8:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764771.311561:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2c8:0x0] 00000008:00010000:3.0:1775764771.311575:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1775764771.311577:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d54b0, osc lock ffff8f9966ace4c0, flags 0x400001002 00010000:00010000:3.0:1775764771.311584:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000d1791e23/0x36f9bdc9e41cad01 lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x5a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eec96 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 162 lru_type: 1 00010000:00010000:3.0:1775764771.311590:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000d1791e23/0x36f9bdc9e41cad01 lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x5a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eec96 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 162 lru_type: 1 00000020:00200000:3.0:1775764771.311597:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1775764771.311599:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000d1791e23/0x36f9bdc9e41cad01 lrc: 6/0,1 mode: PW/PW res: [0x280000403:0x5a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4eec96 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 162 lru_type: 1 00010000:00010000:3.0:1775764771.311604:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000d1791e23/0x36f9bdc9e41cad01 lrc: 5/0,0 mode: PW/PW res: [0x280000403:0x5a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eec96 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 163 lru_type: 1 00000020:00200000:3.0:1775764771.311610:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1775764771.311629:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2c8:0x0] updating i_size 260096 i_blocks 515 00000020:00010000:3.0:1775764771.311633:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_release():245 00000020:00010000:3.0:1775764771.311635:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.311638:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d54b0 (ffff8f9970314758/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.311640:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764771.311642:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d54b0 (ffff8f9970314e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764771.311658:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2c8:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.311676:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.311679:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.311681:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:3.0:1775764771.311683:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.311685:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=16 00000080:00200000:3.0:1775764771.311686:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.311688:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:3.0:1775764771.311689:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.311691:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=16 00000080:00200000:3.0:1775764771.311692:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.311695:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=STUDENTS.PX, flags=0 00000080:00200000:3.0:1775764771.311697:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 3 00000080:00200000:3.0:1775764771.311701:0:19723:0:(namei.c:2395:ll_unlink()) VFS Op:name=STUDENTS.PX, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08) 00010000:00010000:3.0:1775764771.311727:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98c00/0x36f9bdc9e421c515 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c8:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f9ecc expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764771.311756:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2c8:0x0]: losing layout lock 00010000:00010000:3.0:1775764771.311772:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970972600/0x36f9bdc9e421be54 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2c8:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f7f84 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764771.311781:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2c8:0x0]: losing layout lock 00010000:00010000:3.0:1775764771.311804:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98c00/0x36f9bdc9e421c515 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2c8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f9ecc expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764771.311808:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970972600/0x36f9bdc9e421be54 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2c8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f7f84 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764771.311812:0:19723:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1775764771.311813:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98c00/0x36f9bdc9e421c515 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2c8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f9ecc expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764771.311824:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970972600/0x36f9bdc9e421be54 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2c8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f7f84 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000020:00100000:3.0:1775764771.311838:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1775764771.311844:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970683480 x1862022966946688/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764771.311859:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970683480 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966946688:192.168.202.146@tcp:36:dbench.0 00000100:00100000:3.0:1775764771.311914:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413e40 going to sleep for 11 seconds 00000020:00010000:1.0:1775764771.312472:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_release():245 00000020:00010000:1.0:1775764771.312474:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.312477:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f1c7430 (ffff8f99705b8998/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.312479:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764771.312480:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f1c7430 (ffff8f99705b8638/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764771.312493:0:19888:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2da:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764771.312511:0:19888:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2d8:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764771.312515:0:19888:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2d8:0x0] 00000008:00010000:1.0:1775764771.312526:0:19888:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 2b 00000008:00010000:1.0:1775764771.312527:0:19888:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f1c7430, osc lock ffff8f995875db50, flags 0x400041012 00010000:00010000:1.0:1775764771.312535:0:19888:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000022d8547f/0x36f9bdc9e41caf1c lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x62:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef190 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 74 lru_type: 1 00010000:00010000:1.0:1775764771.312541:0:19888:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000022d8547f/0x36f9bdc9e41caf1c lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x62:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef190 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 74 lru_type: 1 00010000:00010000:1.0:1775764771.312547:0:19888:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000022d8547f/0x36f9bdc9e41caf1c lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x62:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ef190 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 74 lru_type: 1 00010000:00010000:1.0:1775764771.312552:0:19888:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000022d8547f/0x36f9bdc9e41caf1c lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x62:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef190 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 75 lru_type: 1 00000020:00010000:1.0:1775764771.312556:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_release():245 00000020:00010000:1.0:1775764771.312557:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.312559:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f1c7430 (ffff8f99705b8998/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.312560:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764771.312562:0:19888:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f1c7430 (ffff8f99705b8638/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764771.312568:0:19888:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2d8:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:3.0:1775764771.316069:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970683480 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966946688:192.168.202.146@tcp:36:dbench.0 00000080:00200000:3.0:1775764771.316190:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.316197:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.316203:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:3.0:1775764771.316209:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.316228:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:3.0:1775764771.316240:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.316244:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:3.0:1775764771.316247:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.316249:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:3.0:1775764771.316250:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.316253:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=CHANGED.DB, flags=1 00010000:00010000:3.0:1775764771.316268:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fcc00/0x36f9bdc9e421ab53 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xec4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f4618 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00010000:3.0:1775764771.316276:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421ab53 from it ffffafbac1c7baf8 00010000:00010000:3.0:1775764771.316290:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fcc00/0x36f9bdc9e421ab53 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xec4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f4618 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1775764771.316297:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fcc00/0x36f9bdc9e421ab53 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xec4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f4618 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00200008:3.0:1775764771.316330:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file CHANGED.DB:[0x200000401:0xec4:0x0](ffff8f9952eb1988), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764771.316336:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xec4:0x0](ffff8f9952eb1988),name=CHANGED.DB 00000002:00010000:3.0:1775764771.316354:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xec4:0x0]) in obj [0x200000401:0x2bb:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764771.316365:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fcc00/0x36f9bdc9e421ab53 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xec4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f4618 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1775764771.316373:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fcc00/0x36f9bdc9e421ab53 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xec4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f4618 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1775764771.316379:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2bb:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764771.316384:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xec4:0x0](ffff8f9952eb1988) 00000080:00010000:3.0:1775764771.316388:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xec4:0x0](ffff8f9952eb1988) for lock 0x36f9bdc9e421ab53 00000080:00010000:3.0:1775764771.316393:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421ab53 from it ffffafbac1c7bcb8 00010000:00010000:3.0:1775764771.316396:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fcc00/0x36f9bdc9e421ab53 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xec4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f4618 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1775764771.316403:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99702fcc00/0x36f9bdc9e421ab53 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xec4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f4618 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 1 00000080:00200000:3.0:1775764771.316451:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xec4:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764771.316460:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xec4:0x0] 00000008:00010000:3.0:1775764771.316488:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1775764771.316491:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d54b0, osc lock ffff8f9966ace4c0, flags 0x400001002 00010000:00010000:3.0:1775764771.316509:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c5083c7f/0x36f9bdc9e421ab29 lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x656:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f459a expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:3.0:1775764771.316517:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c5083c7f/0x36f9bdc9e421ab29 lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x656:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f459a expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00000020:00200000:3.0:1775764771.316525:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1775764771.316528:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c5083c7f/0x36f9bdc9e421ab29 lrc: 6/0,1 mode: PW/PW res: [0x280000403:0x656:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5f459a expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:3.0:1775764771.316533:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c5083c7f/0x36f9bdc9e421ab29 lrc: 5/0,0 mode: PW/PW res: [0x280000403:0x656:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f459a expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00000020:00200000:3.0:1775764771.316539:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1775764771.316546:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0xec4:0x0] updating i_size 4096 i_blocks 1 00000020:00010000:3.0:1775764771.316549:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_release():245 00000020:00010000:3.0:1775764771.316551:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.316552:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d54b0 (ffff8f9970314758/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.316555:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764771.316557:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d54b0 (ffff8f9970314e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764771.316575:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xec4:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.316590:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.316593:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.316595:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:3.0:1775764771.316597:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.316599:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=16 00000080:00200000:3.0:1775764771.316600:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.316602:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:3.0:1775764771.316603:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.316605:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=16 00000080:00200000:3.0:1775764771.316606:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000100:00100000:1.0:1775764771.316795:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970b51c00 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966946304:192.168.202.146@tcp:36:dbench.0 00000080:00200000:1.0:1775764771.316847:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2a5:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764771.316854:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:1.0:1775764771.316865:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:1.0:1775764771.316867:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:1.0:1775764771.316888:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2a5:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764771.317201:0:19724:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2a5:0x0](ffff8f9952d13a88) 00000008:00010000:1.0:1775764771.317229:0:19724:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 1 [cma]time 1775764054 1775764054 0 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000009eadeed7/0x36f9bdc9e41ca807 lrc: 1/0,0 mode: PW/PW res: [0x240000403:0x4c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4edf53 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 58 lru_type: 1 00000080:00200000:1.0:1775764771.317281:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.317283:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.317286:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:1.0:1775764771.317288:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.317290:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:1.0:1775764771.317292:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764771.317294:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:1.0:1775764771.317297:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764771.317299:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=COREL, flags=17 00000080:00200000:1.0:1775764771.317302:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x292:0x0](ffff8f9952d89148), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764771.317313:0:19724:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=GRAPHIC2.CDR, dir=[0x200000401:0x292:0x0](ffff8f9952d89148), flags=1 00000080:00200000:1.0:1775764771.317317:0:19724:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=GRAPHIC2.CDR, dir=[0x200000401:0x292:0x0](ffff8f9952d89148), intent=getattr 00010000:00010000:1.0:1775764771.317327:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4600/0x36f9bdc9e421c570 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fa062 expref: -99 pid: 19889 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:1.0:1775764771.317344:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c570 from it ffffafbac1c5b900 00010000:00010000:1.0:1775764771.317348:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4600/0x36f9bdc9e421c570 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fa062 expref: -99 pid: 19889 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764771.317355:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4600/0x36f9bdc9e421c570 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fa062 expref: -99 pid: 19889 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:1.0:1775764771.317384:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file GRAPHIC2.CDR:[0x200000401:0x2a3:0x0](ffff8f9952d8e3c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764771.317389:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2a3:0x0](ffff8f9952d8e3c8),name=GRAPHIC2.CDR 00000002:00010000:1.0:1775764771.317399:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2a3:0x0]) in obj [0x200000401:0x292:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764771.317407:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093ca00/0x36f9bdc9e4217640 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2a3:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e8ffc expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764771.317415:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093ca00/0x36f9bdc9e4217640 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2a3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e8ffc expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764771.317422:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x292:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764771.317425:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2a3:0x0](ffff8f9952d8e3c8) 00000080:00010000:1.0:1775764771.317428:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2a3:0x0](ffff8f9952d8e3c8) for lock 0x36f9bdc9e4217640 00000080:00010000:1.0:1775764771.317432:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e4217640 from it ffffafbac1c5bcb8 00010000:00010000:1.0:1775764771.317435:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093ca00/0x36f9bdc9e4217640 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5e8ffc expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764771.317443:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093ca00/0x36f9bdc9e4217640 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e8ffc expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764771.317468:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2a3:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764771.317473:0:19724:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2a3:0x0] 00000008:00010000:1.0:1775764771.317489:0:19724:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764771.317492:0:19724:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f1c7430, osc lock ffff8f995875db50, flags 0x400001002 00010000:00010000:1.0:1775764771.317503:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000609c9638/0x36f9bdc9e41ca7ba lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x4b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ede9d expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 57 lru_type: 1 00010000:00010000:1.0:1775764771.317513:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000609c9638/0x36f9bdc9e41ca7ba lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x4b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ede9d expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 57 lru_type: 1 00000020:00200000:1.0:1775764771.317520:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764771.317522:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000609c9638/0x36f9bdc9e41ca7ba lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x4b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ede9d expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 57 lru_type: 1 00010000:00010000:1.0:1775764771.317527:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000609c9638/0x36f9bdc9e41ca7ba lrc: 4/0,0 mode: PW/PW res: [0x240000403:0x4b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ede9d expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 58 lru_type: 1 00000020:00200000:1.0:1775764771.317533:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764771.317536:0:19724:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2a3:0x0] updating i_size 260096 i_blocks 1 00000020:00010000:1.0:1775764771.317538:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_release():245 00000020:00010000:1.0:1775764771.317540:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.317542:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f1c7430 (ffff8f99705b8998/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764771.317544:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705fafe8 (ffff8f99705b8638/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764771.317545:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f1c7430 (ffff8f99705b8638/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764771.317555:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2a3:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764771.317568:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.317570:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.317572:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:1.0:1775764771.317574:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.317576:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=16 00000080:00200000:1.0:1775764771.317577:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764771.317579:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:1.0:1775764771.317580:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764771.317582:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=COREL, flags=16 00000080:00200000:1.0:1775764771.317583:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x292:0x0](ffff8f9952d89148), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764771.317585:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=GRAPHIC2.CDR, flags=0 00000080:00200000:1.0:1775764771.317588:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x292:0x0](ffff8f9952d89148), inode mode 41c0 mask 3 00000080:00200000:1.0:1775764771.317591:0:19724:0:(namei.c:2395:ll_unlink()) VFS Op:name=GRAPHIC2.CDR, dir=[0x200000401:0x292:0x0](ffff8f9952d89148) 00010000:00010000:0.0:1775764771.318715:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4600/0x36f9bdc9e421c570 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a3:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5fa062 expref: -99 pid: 19889 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1775764771.318780:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2a3:0x0]: losing layout lock 00010000:00010000:0.0:1775764771.318813:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093ca00/0x36f9bdc9e4217640 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a3:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5e8ffc expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764771.318831:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2a3:0x0]: losing layout lock 00010000:00010000:0.0:1775764771.318866:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4600/0x36f9bdc9e421c570 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2a3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fa062 expref: -99 pid: 19889 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764771.318872:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093ca00/0x36f9bdc9e4217640 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2a3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5e8ffc expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764771.318876:0:19724:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1775764771.318879:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4600/0x36f9bdc9e421c570 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2a3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fa062 expref: -99 pid: 19889 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764771.318892:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093ca00/0x36f9bdc9e4217640 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2a3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5e8ffc expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000020:00100000:0.0:1775764771.318912:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1775764771.318921:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f0c8a80 x1862022966946944/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764771.318953:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c8a80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966946944:192.168.202.146@tcp:36:dbench.0 00000100:00100000:0.0:1775764771.319042:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f6c0 going to sleep for 11 seconds 00000100:00100000:0.0:1775764771.322858:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c8a80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966946944:192.168.202.146@tcp:36:dbench.0 00000080:00200000:0.0:1775764771.322939:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2a3:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764771.322952:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1775764771.322974:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:0.0:1775764771.322977:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:0.0:1775764771.323023:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2a3:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764771.323367:0:19724:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2a3:0x0](ffff8f9952d8e3c8) 00000008:00010000:0.0:1775764771.323800:0:19724:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 1 [cma]time 1775764054 1775764054 0 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000609c9638/0x36f9bdc9e41ca7ba lrc: 1/0,0 mode: PW/PW res: [0x240000403:0x4b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ede9d expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 58 lru_type: 1 00000080:00200000:0.0:1775764771.323888:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.323895:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.323901:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:0.0:1775764771.323907:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.323911:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:0.0:1775764771.323913:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.323916:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:0.0:1775764771.323918:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.323921:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=COREL, flags=17 00000080:00200000:0.0:1775764771.323924:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x292:0x0](ffff8f9952d89148), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.323937:0:19724:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=CORELDRW.CDT, dir=[0x200000401:0x292:0x0](ffff8f9952d89148), flags=1 00000080:00200000:0.0:1775764771.323942:0:19724:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=CORELDRW.CDT, dir=[0x200000401:0x292:0x0](ffff8f9952d89148), intent=getattr 00010000:00010000:0.0:1775764771.323961:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c577 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fa07e expref: -99 pid: 19889 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:0.0:1775764771.323975:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c577 from it ffffafbac1c5b900 00010000:00010000:0.0:1775764771.323979:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c577 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fa07e expref: -99 pid: 19889 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764771.323988:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c577 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fa07e expref: -99 pid: 19889 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:0.0:1775764771.324203:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file CORELDRW.CDT:[0x200000401:0x2a7:0x0](ffff8f9952d100c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764771.324209:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2a7:0x0](ffff8f9952d100c8),name=CORELDRW.CDT 00000002:00010000:0.0:1775764771.324227:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2a7:0x0]) in obj [0x200000401:0x292:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764771.324238:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093de00/0x36f9bdc9e421761d lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2a7:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e8f77 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764771.324249:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093de00/0x36f9bdc9e421761d lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2a7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e8f77 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764771.324256:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x292:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764771.324262:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2a7:0x0](ffff8f9952d100c8) 00000080:00010000:0.0:1775764771.324265:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2a7:0x0](ffff8f9952d100c8) for lock 0x36f9bdc9e421761d 00000080:00010000:0.0:1775764771.324271:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421761d from it ffffafbac1c5bcb8 00010000:00010000:0.0:1775764771.324274:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093de00/0x36f9bdc9e421761d lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5e8f77 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764771.324281:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093de00/0x36f9bdc9e421761d lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e8f77 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764771.324387:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2a7:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1775764771.324464:0:19724:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2a7:0x0] 00000008:00010000:0.0:1775764771.324490:0:19724:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1775764771.324494:0:19724:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0df830, osc lock ffff8f996686ee40, flags 0x400001002 00010000:00010000:0.0:1775764771.324509:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c6ec0b7f/0x36f9bdc9e41ca84d lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x4d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4edffb expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 70 lru_type: 1 00010000:00010000:0.0:1775764771.324525:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c6ec0b7f/0x36f9bdc9e41ca84d lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x4d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4edffb expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 70 lru_type: 1 00000020:00200000:0.0:1775764771.324536:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1775764771.324539:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c6ec0b7f/0x36f9bdc9e41ca84d lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x4d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4edffb expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 70 lru_type: 1 00010000:00010000:0.0:1775764771.324547:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c6ec0b7f/0x36f9bdc9e41ca84d lrc: 4/0,0 mode: PW/PW res: [0x240000403:0x4d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4edffb expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 71 lru_type: 1 00000020:00200000:0.0:1775764771.324558:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1775764771.324566:0:19724:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2a7:0x0] updating i_size 260096 i_blocks 1 00000020:00010000:0.0:1775764771.324570:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970b8b908 (ffff8f9966dc2f38/0) at cl_lock_release():245 00000020:00010000:0.0:1775764771.324573:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970b8b908 (ffff8f9966dc2f38/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764771.324576:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0df830 (ffff8f9966dc2998/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764771.324579:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970b8b908 (ffff8f9966dc2f38/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764771.324582:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0df830 (ffff8f9966dc2f38/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764771.324597:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2a7:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764771.324629:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.324634:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.324637:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:0.0:1775764771.324641:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.324643:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=16 00000080:00200000:0.0:1775764771.324645:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.324648:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:0.0:1775764771.324651:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.324654:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=COREL, flags=16 00000080:00200000:0.0:1775764771.324656:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x292:0x0](ffff8f9952d89148), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.324661:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=CORELDRW.CDT, flags=0 00000080:00200000:0.0:1775764771.324664:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x292:0x0](ffff8f9952d89148), inode mode 41c0 mask 3 00000080:00200000:0.0:1775764771.324669:0:19724:0:(namei.c:2395:ll_unlink()) VFS Op:name=CORELDRW.CDT, dir=[0x200000401:0x292:0x0](ffff8f9952d89148) 00010000:00010000:0.0:1775764771.324708:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c577 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a7:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5fa07e expref: -99 pid: 19889 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1775764771.324729:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2a7:0x0]: losing layout lock 00010000:00010000:0.0:1775764771.324810:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093de00/0x36f9bdc9e421761d lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a7:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5e8f77 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764771.324828:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2a7:0x0]: losing layout lock 00010000:00010000:0.0:1775764771.324862:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c577 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2a7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fa07e expref: -99 pid: 19889 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764771.324869:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093de00/0x36f9bdc9e421761d lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2a7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5e8f77 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764771.324875:0:19724:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1775764771.324877:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c577 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2a7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fa07e expref: -99 pid: 19889 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764771.324890:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f997093de00/0x36f9bdc9e421761d lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2a7:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5e8f77 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000020:00100000:0.0:1775764771.324912:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1775764771.324922:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f0cb480 x1862022966947456/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764771.324949:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0cb480 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966947456:192.168.202.146@tcp:36:dbench.0 00000100:00100000:0.0:1775764771.325118:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f6c0 going to sleep for 11 seconds 00000100:00100000:3.0:1775764771.339162:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966947968:192.168.202.146@tcp:36:dbench.0 00000080:00200000:3.0:1775764771.339214:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x295:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1775764771.339224:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764771.339242:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:3.0:1775764771.339243:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:3.0:1775764771.339270:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x295:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.339586:0:19724:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x295:0x0](ffff8f9952d66c08) 00000008:00010000:3.0:1775764771.339633:0:19724:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 515 [cma]time 1775764731 1775764731 0 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001cab28fe/0x36f9bdc9e41ca5de lrc: 1/0,0 mode: PW/PW res: [0x240000403:0x44:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eda36 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 93 lru_type: 1 00000080:00200000:3.0:1775764771.339706:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.339710:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.339714:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:3.0:1775764771.339717:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00010000:00010000:1.0:1775764771.339931:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1775764771.340332:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000100:00100000:3.0:1775764771.343986:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966948224:192.168.202.146@tcp:36:dbench.0 00000080:00200000:3.0:1775764771.344035:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.344038:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.344041:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:3.0:1775764771.344043:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.344046:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:3.0:1775764771.344047:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.344050:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:3.0:1775764771.344051:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.344054:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:3.0:1775764771.344056:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.344061:0:19723:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=ENROLL.DB, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), flags=1 00000080:00200000:3.0:1775764771.344064:0:19723:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=ENROLL.DB, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), intent=getattr 00010000:00010000:3.0:1775764771.344072:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98400/0x36f9bdc9e421c52a lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2d6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9f20 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:3.0:1775764771.344081:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c52a from it ffffafbac1c7b900 00010000:00010000:3.0:1775764771.344083:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98400/0x36f9bdc9e421c52a lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2d6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9f20 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764771.344088:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98400/0x36f9bdc9e421c52a lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9f20 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:3.0:1775764771.344102:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file ENROLL.DB:[0x200000401:0x2d6:0x0](ffff8f9952d96c08), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764771.344105:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2d6:0x0](ffff8f9952d96c08),name=ENROLL.DB 00000002:00010000:3.0:1775764771.344111:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2d6:0x0]) in obj [0x200000401:0x2bb:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764771.344116:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea200/0x36f9bdc9e421bf42 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2d6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8263 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.344122:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea200/0x36f9bdc9e421bf42 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2d6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8263 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764771.344129:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2bb:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764771.344132:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2d6:0x0](ffff8f9952d96c08) 00000080:00010000:3.0:1775764771.344134:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2d6:0x0](ffff8f9952d96c08) for lock 0x36f9bdc9e421bf42 00000080:00010000:3.0:1775764771.344138:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421bf42 from it ffffafbac1c7bcb8 00010000:00010000:3.0:1775764771.344141:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea200/0x36f9bdc9e421bf42 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2d6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8263 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.344152:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea200/0x36f9bdc9e421bf42 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8263 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764771.344182:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2d6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764771.344194:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2d6:0x0] 00000008:00010000:3.0:1775764771.344218:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1775764771.344221:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d54b0, osc lock ffff8f9966ace4c0, flags 0x400001002 00010000:00010000:3.0:1775764771.344234:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000260038e6/0x36f9bdc9e41caed6 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x61:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef0e8 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 101 lru_type: 1 00010000:00010000:3.0:1775764771.344242:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000260038e6/0x36f9bdc9e41caed6 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x61:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef0e8 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 101 lru_type: 1 00000020:00200000:3.0:1775764771.344251:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1775764771.344253:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000260038e6/0x36f9bdc9e41caed6 lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x61:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ef0e8 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 101 lru_type: 1 00010000:00010000:3.0:1775764771.344258:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000260038e6/0x36f9bdc9e41caed6 lrc: 4/0,0 mode: PW/PW res: [0x280000403:0x61:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef0e8 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 102 lru_type: 1 00000020:00200000:3.0:1775764771.344265:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1775764771.344271:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2d6:0x0] updating i_size 260096 i_blocks 1 00000020:00010000:3.0:1775764771.344274:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_release():245 00000020:00010000:3.0:1775764771.344276:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.344278:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d54b0 (ffff8f9970314758/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.344281:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764771.344283:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d54b0 (ffff8f9970314e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764771.344319:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2d6:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.344337:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.344341:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.344344:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:3.0:1775764771.344348:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.344350:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=16 00000080:00200000:3.0:1775764771.344353:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.344355:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:3.0:1775764771.344358:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.344360:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=16 00000080:00200000:3.0:1775764771.344362:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.344366:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=ENROLL.DB, flags=0 00000080:00200000:3.0:1775764771.344369:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 3 00000080:00200000:3.0:1775764771.344372:0:19723:0:(namei.c:2395:ll_unlink()) VFS Op:name=ENROLL.DB, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08) 00010000:00010000:3.0:1775764771.344403:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98400/0x36f9bdc9e421c52a lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f9f20 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764771.344425:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2d6:0x0]: losing layout lock 00010000:00010000:3.0:1775764771.344446:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea200/0x36f9bdc9e421bf42 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f8263 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764771.344460:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2d6:0x0]: losing layout lock 00010000:00010000:3.0:1775764771.344487:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98400/0x36f9bdc9e421c52a lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2d6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f9f20 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764771.344492:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea200/0x36f9bdc9e421bf42 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2d6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f8263 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764771.344499:0:19723:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1775764771.344501:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98400/0x36f9bdc9e421c52a lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2d6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f9f20 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764771.344508:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea200/0x36f9bdc9e421bf42 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2d6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f8263 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000020:00100000:3.0:1775764771.344523:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1775764771.344528:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970681880 x1862022966948480/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764771.344543:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970681880 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966948480:192.168.202.146@tcp:36:dbench.0 00000100:00100000:3.0:1775764771.344641:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413e40 going to sleep for 11 seconds 00000100:00100000:3.0:1775764771.356630:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970681880 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966948480:192.168.202.146@tcp:36:dbench.0 00000080:00200000:3.0:1775764771.356702:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.356706:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.356720:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:3.0:1775764771.356723:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.356726:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:3.0:1775764771.356729:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.356732:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:3.0:1775764771.356734:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.356762:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:3.0:1775764771.356766:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.356893:0:19723:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=STUDENTS.XG0, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), flags=1 00000080:00200000:3.0:1775764771.356898:0:19723:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=STUDENTS.XG0, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), intent=getattr 00010000:00010000:3.0:1775764771.356910:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98e00/0x36f9bdc9e421c531 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2ca:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9f3c expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:3.0:1775764771.356922:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c531 from it ffffafbac1c7b900 00010000:00010000:3.0:1775764771.356925:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98e00/0x36f9bdc9e421c531 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2ca:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9f3c expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764771.356933:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98e00/0x36f9bdc9e421c531 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ca:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9f3c expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:3.0:1775764771.357095:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file STUDENTS.XG0:[0x200000401:0x2ca:0x0](ffff8f9952e2f448), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764771.357107:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2ca:0x0](ffff8f9952e2f448),name=STUDENTS.XG0 00000002:00010000:3.0:1775764771.357114:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2ca:0x0]) in obj [0x200000401:0x2bb:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764771.357120:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970973200/0x36f9bdc9e421be77 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2ca:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8002 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.357138:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970973200/0x36f9bdc9e421be77 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2ca:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8002 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764771.357145:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2bb:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764771.357149:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2ca:0x0](ffff8f9952e2f448) 00000080:00010000:3.0:1775764771.357151:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2ca:0x0](ffff8f9952e2f448) for lock 0x36f9bdc9e421be77 00000080:00010000:3.0:1775764771.357155:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421be77 from it ffffafbac1c7bcb8 00010000:00010000:3.0:1775764771.357158:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970973200/0x36f9bdc9e421be77 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2ca:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8002 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.357181:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970973200/0x36f9bdc9e421be77 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ca:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8002 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764771.357216:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2ca:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764771.357221:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2ca:0x0] 00000008:00010000:3.0:1775764771.357237:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1775764771.357238:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d54b0, osc lock ffff8f9966ace4c0, flags 0x400001002 00010000:00010000:3.0:1775764771.357248:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000003d943106/0x36f9bdc9e420da09 lrc: 2/1,0 mode: PR/PR res: [0x280000403:0x5b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5c98dc expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 30 lru_type: 1 00010000:00010000:3.0:1775764771.357255:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000003d943106/0x36f9bdc9e420da09 lrc: 2/1,0 mode: PR/PR res: [0x280000403:0x5b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5c98dc expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 30 lru_type: 1 00000020:00200000:3.0:1775764771.357284:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1775764771.357291:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000003d943106/0x36f9bdc9e420da09 lrc: 5/1,0 mode: PR/PR res: [0x280000403:0x5b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5c98dc expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 30 lru_type: 1 00010000:00010000:3.0:1775764771.357298:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000003d943106/0x36f9bdc9e420da09 lrc: 4/0,0 mode: PR/PR res: [0x280000403:0x5b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5c98dc expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 31 lru_type: 1 00000020:00200000:3.0:1775764771.357308:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1775764771.357325:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2ca:0x0] updating i_size 260096 i_blocks 515 00000020:00010000:3.0:1775764771.357329:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_release():245 00000020:00010000:3.0:1775764771.357332:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.357334:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d54b0 (ffff8f9970314758/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.357337:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef50000 (ffff8f9970314e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764771.357340:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d54b0 (ffff8f9970314e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764771.357377:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2ca:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.357415:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.357419:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.357422:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:3.0:1775764771.357426:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.357429:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=16 00000080:00200000:3.0:1775764771.357431:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.357433:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:3.0:1775764771.357435:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.357452:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=16 00000080:00200000:3.0:1775764771.357455:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.357459:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=STUDENTS.XG0, flags=0 00000080:00200000:3.0:1775764771.357461:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 3 00000080:00200000:3.0:1775764771.357464:0:19723:0:(namei.c:2395:ll_unlink()) VFS Op:name=STUDENTS.XG0, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08) 00010000:00010000:3.0:1775764771.357488:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98e00/0x36f9bdc9e421c531 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ca:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f9f3c expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764771.357500:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2ca:0x0]: losing layout lock 00010000:00010000:3.0:1775764771.357513:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970973200/0x36f9bdc9e421be77 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ca:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f8002 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764771.357522:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2ca:0x0]: losing layout lock 00010000:00010000:3.0:1775764771.357545:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98e00/0x36f9bdc9e421c531 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2ca:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f9f3c expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764771.357551:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970973200/0x36f9bdc9e421be77 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2ca:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f8002 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764771.357554:0:19723:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1775764771.357556:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98e00/0x36f9bdc9e421c531 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2ca:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f9f3c expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764771.357566:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970973200/0x36f9bdc9e421be77 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2ca:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f8002 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000020:00100000:3.0:1775764771.357581:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1775764771.357587:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970680a80 x1862022966948992/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764771.357620:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970680a80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966948992:192.168.202.146@tcp:36:dbench.0 00000100:00100000:3.0:1775764771.357704:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413e40 going to sleep for 11 seconds 00010000:00010000:0.0:1775764771.360143:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0da00/0x36f9bdc9e4217250 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x29a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5e83a0 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764771.360184:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x29a:0x0]: losing layout lock 00010000:00010000:0.0:1775764771.360225:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c58c lrc: 1/0,0 mode: --/PR res: [0x200000401:0x29a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fa0d2 expref: -99 pid: 19889 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764771.360232:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0da00/0x36f9bdc9e4217250 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x29a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5e83a0 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764771.360237:0:19724:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1775764771.360240:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c58c lrc: 0/0,0 mode: --/PR res: [0x200000401:0x29a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fa0d2 expref: -99 pid: 19889 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764771.360252:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed0da00/0x36f9bdc9e4217250 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x29a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5e83a0 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000020:00100000:0.0:1775764771.360270:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 2 is allocated opc 36, max 7 00000100:00100000:0.0:1775764771.360282:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f0cb800 x1862022966949248/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764771.360314:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0cb800 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966949248:192.168.202.146@tcp:36:dbench.0 00000100:00100000:0.0:1775764771.360432:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f240 going to sleep for 11 seconds 00000100:00100000:3.0:1775764771.372344:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970680a80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966948992:192.168.202.146@tcp:36:dbench.0 00000080:00200000:3.0:1775764771.372420:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.372427:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.372433:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:3.0:1775764771.372440:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.372443:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:3.0:1775764771.372446:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.372450:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:3.0:1775764771.372453:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.372456:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:3.0:1775764771.372458:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.372475:0:19723:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=ENROLL.VAL, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), flags=1 00000080:00200000:3.0:1775764771.372480:0:19723:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=ENROLL.VAL, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), intent=getattr 00010000:00010000:3.0:1775764771.372503:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99000/0x36f9bdc9e421c538 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2de:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9f58 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:3.0:1775764771.372520:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c538 from it ffffafbac1c7b900 00010000:00010000:3.0:1775764771.372525:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99000/0x36f9bdc9e421c538 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2de:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9f58 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764771.372534:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99000/0x36f9bdc9e421c538 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2de:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9f58 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:3.0:1775764771.372564:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file ENROLL.VAL:[0x200000401:0x2de:0x0](ffff8f9952dd21c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764771.372570:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2de:0x0](ffff8f9952dd21c8),name=ENROLL.VAL 00000002:00010000:3.0:1775764771.372587:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2de:0x0]) in obj [0x200000401:0x2bb:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764771.372600:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8d200/0x36f9bdc9e421bff1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2de:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8469 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.372612:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8d200/0x36f9bdc9e421bff1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2de:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8469 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764771.372630:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2bb:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764771.372635:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2de:0x0](ffff8f9952dd21c8) 00000080:00010000:3.0:1775764771.372638:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2de:0x0](ffff8f9952dd21c8) for lock 0x36f9bdc9e421bff1 00000080:00010000:3.0:1775764771.372643:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421bff1 from it ffffafbac1c7bcb8 00010000:00010000:3.0:1775764771.372646:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8d200/0x36f9bdc9e421bff1 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2de:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8469 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.372652:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8d200/0x36f9bdc9e421bff1 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2de:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8469 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764771.372700:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2de:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764771.372712:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2de:0x0] 00000100:00100000:3.0:1775764771.380681:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966949504:192.168.202.146@tcp:36:dbench.0 00000080:00200000:3.0:1775764771.380730:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.380734:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.380766:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:3.0:1775764771.380770:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.380774:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:3.0:1775764771.380777:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.380780:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:3.0:1775764771.380782:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.380785:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:3.0:1775764771.380788:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.380801:0:19723:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=STUDENTS.TV, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), flags=1 00000080:00200000:3.0:1775764771.380805:0:19723:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=STUDENTS.TV, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), intent=getattr 00010000:00010000:3.0:1775764771.380816:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98a00/0x36f9bdc9e421c53f lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2e0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9f74 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:3.0:1775764771.380828:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c53f from it ffffafbac1c7b900 00010000:00010000:3.0:1775764771.380831:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98a00/0x36f9bdc9e421c53f lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2e0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9f74 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764771.380838:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98a00/0x36f9bdc9e421c53f lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9f74 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:3.0:1775764771.380855:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file STUDENTS.TV:[0x200000401:0x2e0:0x0](ffff8f9952dd5348), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764771.380859:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2e0:0x0](ffff8f9952dd5348),name=STUDENTS.TV 00000002:00010000:3.0:1775764771.380867:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2e0:0x0]) in obj [0x200000401:0x2bb:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764771.380875:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d48a00/0x36f9bdc9e421c04c lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2e0:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8581 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.380888:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d48a00/0x36f9bdc9e421c04c lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2e0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8581 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764771.380894:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2bb:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764771.380898:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2e0:0x0](ffff8f9952dd5348) 00000080:00010000:3.0:1775764771.380901:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2e0:0x0](ffff8f9952dd5348) for lock 0x36f9bdc9e421c04c 00000080:00010000:3.0:1775764771.380905:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c04c from it ffffafbac1c7bcb8 00010000:00010000:3.0:1775764771.380908:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d48a00/0x36f9bdc9e421c04c lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2e0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8581 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.380914:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d48a00/0x36f9bdc9e421c04c lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8581 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764771.380949:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2e0:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764771.380954:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2e0:0x0] 00000008:00010000:3.0:1775764771.380974:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1775764771.380976:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d55b0, osc lock ffff8f9966ace4c0, flags 0x400001002 00010000:00010000:3.0:1775764771.380986:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000e275e2c0/0x36f9bdc9e420ed6c lrc: 2/1,0 mode: PR/PR res: [0x280000403:0x66:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5cd375 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:3.0:1775764771.380995:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000e275e2c0/0x36f9bdc9e420ed6c lrc: 2/1,0 mode: PR/PR res: [0x280000403:0x66:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5cd375 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00000020:00200000:3.0:1775764771.381004:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1775764771.381007:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000e275e2c0/0x36f9bdc9e420ed6c lrc: 5/1,0 mode: PR/PR res: [0x280000403:0x66:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5cd375 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:3.0:1775764771.381020:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000e275e2c0/0x36f9bdc9e420ed6c lrc: 4/0,0 mode: PR/PR res: [0x280000403:0x66:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5cd375 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00000020:00200000:3.0:1775764771.381028:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1775764771.381033:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2e0:0x0] updating i_size 260096 i_blocks 515 00000020:00010000:3.0:1775764771.381036:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef52fe8 (ffff8f9970314518/0) at cl_lock_release():245 00000020:00010000:3.0:1775764771.381039:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef52fe8 (ffff8f9970314518/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.381042:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d55b0 (ffff8f9970314f38/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.381045:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef52fe8 (ffff8f9970314518/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764771.381047:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d55b0 (ffff8f9970314518/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764771.381065:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2e0:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.381082:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.381086:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.381089:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:3.0:1775764771.381112:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.381117:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=16 00000100:00100000:0.0:1775764771.400874:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efddc00 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966950272:192.168.202.146@tcp:36:dbench.0 00000080:00200000:0.0:1775764771.400950:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x294:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764771.400962:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1775764771.400979:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:0.0:1775764771.400981:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:0.0:1775764771.401016:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x294:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764771.401403:0:19724:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x294:0x0](ffff8f9952d89988) 00000008:00010000:0.0:1775764771.401441:0:19724:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 515 [cma]time 1775764731 1775764731 0 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000031ed7a52/0x36f9bdc9e41ca591 lrc: 1/0,0 mode: PW/PW res: [0x280000403:0x43:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ed95d expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 93 lru_type: 1 00000080:00200000:0.0:1775764771.401522:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.401529:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.401535:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:0.0:1775764771.401542:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.401546:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:0.0:1775764771.401549:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.401552:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:0.0:1775764771.401556:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.401559:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=COREL, flags=17 00000080:00200000:0.0:1775764771.401561:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x292:0x0](ffff8f9952d89148), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.401573:0:19724:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=ARTISTIC.ACL, dir=[0x200000401:0x292:0x0](ffff8f9952d89148), flags=1 00000080:00200000:0.0:1775764771.401577:0:19724:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=ARTISTIC.ACL, dir=[0x200000401:0x292:0x0](ffff8f9952d89148), intent=getattr 00010000:00010000:0.0:1775764771.401593:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c5c4 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x298:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fa157 expref: -99 pid: 19889 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:0.0:1775764771.401603:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c5c4 from it ffffafbac1c5b900 00010000:00010000:0.0:1775764771.401607:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c5c4 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x298:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fa157 expref: -99 pid: 19889 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764771.401631:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c5c4 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x298:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fa157 expref: -99 pid: 19889 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:0.0:1775764771.401664:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file ARTISTIC.ACL:[0x200000401:0x298:0x0](ffff8f9952dc8908), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764771.401670:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x298:0x0](ffff8f9952dc8908),name=ARTISTIC.ACL 00000002:00010000:0.0:1775764771.401684:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x298:0x0]) in obj [0x200000401:0x292:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764771.401693:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971079a00/0x36f9bdc9e421718c lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x298:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e813f expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764771.401703:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971079a00/0x36f9bdc9e421718c lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x298:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e813f expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764771.401709:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x292:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764771.401713:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x298:0x0](ffff8f9952dc8908) 00000080:00010000:0.0:1775764771.401716:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x298:0x0](ffff8f9952dc8908) for lock 0x36f9bdc9e421718c 00000080:00010000:0.0:1775764771.403220:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421718c from it ffffafbac1c5bcb8 00010000:00010000:0.0:1775764771.403228:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971079a00/0x36f9bdc9e421718c lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x298:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5e813f expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764771.403238:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971079a00/0x36f9bdc9e421718c lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x298:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e813f expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764771.403287:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x298:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1775764771.403295:0:19724:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x298:0x0] 00000008:00010000:0.0:1775764771.403320:0:19724:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1775764771.403322:0:19724:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0df830, osc lock ffff8f996686ee40, flags 0x400001002 00010000:00010000:0.0:1775764771.403341:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000139dd483/0x36f9bdc9e4210797 lrc: 2/1,0 mode: PR/PR res: [0x280000403:0x45:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5d270c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:0.0:1775764771.403359:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000139dd483/0x36f9bdc9e4210797 lrc: 2/1,0 mode: PR/PR res: [0x280000403:0x45:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5d270c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00000020:00200000:0.0:1775764771.403368:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1775764771.403371:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000139dd483/0x36f9bdc9e4210797 lrc: 5/1,0 mode: PR/PR res: [0x280000403:0x45:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5d270c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:0.0:1775764771.403379:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000139dd483/0x36f9bdc9e4210797 lrc: 4/0,0 mode: PR/PR res: [0x280000403:0x45:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5d270c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00000020:00200000:0.0:1775764771.403388:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1775764771.403395:0:19724:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x298:0x0] updating i_size 260096 i_blocks 515 00000020:00010000:0.0:1775764771.403399:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970b8b908 (ffff8f9966dc2f38/0) at cl_lock_release():245 00000020:00010000:0.0:1775764771.403401:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970b8b908 (ffff8f9966dc2f38/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764771.403404:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0df830 (ffff8f9966dc2998/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764771.403407:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970b8b908 (ffff8f9966dc2f38/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764771.403409:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0df830 (ffff8f9966dc2f38/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764771.403431:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x298:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764771.403462:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.403467:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.403470:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:0.0:1775764771.403474:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.403477:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=16 00000080:00200000:0.0:1775764771.403480:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.403482:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:0.0:1775764771.403486:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.403488:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=COREL, flags=16 00000080:00200000:0.0:1775764771.403490:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x292:0x0](ffff8f9952d89148), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.403496:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=ARTISTIC.ACL, flags=0 00000080:00200000:0.0:1775764771.403499:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x292:0x0](ffff8f9952d89148), inode mode 41c0 mask 3 00000080:00200000:0.0:1775764771.403504:0:19724:0:(namei.c:2395:ll_unlink()) VFS Op:name=ARTISTIC.ACL, dir=[0x200000401:0x292:0x0](ffff8f9952d89148) 00010000:00010000:0.0:1775764771.403542:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c5c4 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x298:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5fa157 expref: -99 pid: 19889 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1775764771.403563:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x298:0x0]: losing layout lock 00010000:00010000:0.0:1775764771.403581:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971079a00/0x36f9bdc9e421718c lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x298:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5e813f expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764771.403598:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x298:0x0]: losing layout lock 00010000:00010000:0.0:1775764771.403641:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c5c4 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x298:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fa157 expref: -99 pid: 19889 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764771.403649:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971079a00/0x36f9bdc9e421718c lrc: 1/0,0 mode: --/PR res: [0x200000401:0x298:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5e813f expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764771.403655:0:19724:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1775764771.403657:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c5c4 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x298:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fa157 expref: -99 pid: 19889 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764771.403670:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971079a00/0x36f9bdc9e421718c lrc: 0/0,0 mode: --/PR res: [0x200000401:0x298:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5e813f expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000020:00100000:0.0:1775764771.403686:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 2 is allocated opc 36, max 7 00000100:00100000:0.0:1775764771.403695:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f0c8700 x1862022966950784/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764771.403727:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c8700 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966950784:192.168.202.146@tcp:36:dbench.0 00000100:00100000:0.0:1775764771.403824:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f180 going to sleep for 11 seconds 00000100:00100000:0.0:1775764771.414945:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c8700 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966950784:192.168.202.146@tcp:36:dbench.0 00000080:00200000:0.0:1775764771.415036:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x298:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764771.415048:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1775764771.415068:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:0.0:1775764771.415070:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:0.0:1775764771.415103:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x298:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764771.415456:0:19724:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x298:0x0](ffff8f9952dc8908) 00000008:00010000:0.0:1775764771.415497:0:19724:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 260096 blocks 515 [cma]time: 1775764053 1775764053 0 by oinfo size 260096 blocks 515 [cma]time 1775764053 1775764053 1775764673 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000139dd483/0x36f9bdc9e4210797 lrc: 1/0,0 mode: PR/PR res: [0x280000403:0x45:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5d270c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00000080:00200000:0.0:1775764771.415565:0:19724:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x292:0x0](ffff8f9952d89148) pos/size18446744073709551614/7168 32bit_api 0 00000080:00200008:0.0:1775764771.415582:0:19724:0:(file.c:447:ll_file_release()) START file COREL:[0x200000401:0x292:0x0](ffff8f9952d89148), flags 304000 00010000:00010000:0.0:1775764771.415606:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971079200/0x36f9bdc9e41ca56e lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x292:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ed8fb expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00000080:00200000:0.0:1775764771.415660:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.415666:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.415671:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:0.0:1775764771.415676:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.415680:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:0.0:1775764771.415682:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.415685:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:0.0:1775764771.415687:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.415692:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=ACCESS, flags=1 00010000:00010000:0.0:1775764771.415707:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5c00/0x36f9bdc9e421c267 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8bc4 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00010000:0.0:1775764771.415717:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c267 from it ffffafbac1c5baf8 00010000:00010000:0.0:1775764771.415720:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5c00/0x36f9bdc9e421c267 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8bc4 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764771.415729:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5c00/0x36f9bdc9e421c267 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8bc4 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200008:0.0:1775764771.415799:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file ACCESS:[0x200000401:0x2e1:0x0](ffff8f9952d91148), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764771.415805:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2e1:0x0](ffff8f9952d91148),name=ACCESS 00000002:00010000:0.0:1775764771.415820:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2e1:0x0]) in obj [0x200000401:0x26b:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764771.415828:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5c00/0x36f9bdc9e421c267 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8bc4 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764771.415837:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5c00/0x36f9bdc9e421c267 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8bc4 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1775764771.415844:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26b:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764771.415849:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2e1:0x0](ffff8f9952d91148) 00000080:00010000:0.0:1775764771.415852:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2e1:0x0](ffff8f9952d91148) for lock 0x36f9bdc9e421c267 00000080:00010000:0.0:1775764771.415857:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c267 from it ffffafbac1c5bcb8 00010000:00010000:0.0:1775764771.415859:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5c00/0x36f9bdc9e421c267 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8bc4 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764771.415865:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5c00/0x36f9bdc9e421c267 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8bc4 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:0.0:1775764771.415894:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.415897:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.415900:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=19 00000080:00200000:0.0:1775764771.415902:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.415904:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=19 00000080:00200000:0.0:1775764771.415906:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.415909:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=19 00000080:00200000:0.0:1775764771.415911:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.415913:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=ACCESS, flags=259 00000080:00200000:0.0:1775764771.415917:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2e1:0x0](ffff8f9952d91148), inode mode 41c0 mask 44 00000080:00200008:0.0:1775764771.415922:0:19724:0:(file.c:1057:ll_file_open()) START file ACCESS:[0x200000401:0x2e1:0x0](ffff8f9952d91148), flags 304000 00000080:00200008:0.0:1775764771.415936:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file ACCESS:[0x200000401:0x2e1:0x0](ffff8f9952d91148), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764771.415939:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2e1:0x0](ffff8f9952d91148),name=ACCESS 00000002:00010000:0.0:1775764771.415945:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2e1:0x0]) in obj [0x200000401:0x26b:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764771.415950:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5c00/0x36f9bdc9e421c267 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8bc4 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764771.415957:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5c00/0x36f9bdc9e421c267 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8bc4 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:0.0:1775764771.415962:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26b:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764771.415965:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2e1:0x0](ffff8f9952d91148) 00000080:00010000:0.0:1775764771.415968:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2e1:0x0](ffff8f9952d91148) for lock 0x36f9bdc9e421c267 00000080:00010000:0.0:1775764771.415971:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c267 from it ffffafbac1c5bcd8 00010000:00010000:0.0:1775764771.415974:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5c00/0x36f9bdc9e421c267 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8bc4 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764771.415985:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5c00/0x36f9bdc9e421c267 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8bc4 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:0.0:1775764771.415997:0:19724:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2e1:0x0](ffff8f9952d91148) pos/size0/7168 32bit_api 0 00000002:00010000:0.0:1775764771.416007:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2e1:0x0]) in obj [0x200000401:0x2e1:0x0], intent: readdir flags 0 00010000:00010000:0.0:1775764771.416011:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5c00/0x36f9bdc9e421c267 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8bc4 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:0.0:1775764771.416019:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5c00/0x36f9bdc9e421c267 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8bc4 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000100:00100000:0.0:1775764771.416574:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0ca300 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966951168:192.168.202.146@tcp:37:dbench.0 00000100:00100000:0.0:1775764771.416662:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f180 going to sleep for 11 seconds 00000100:00100000:3.0:1775764771.432646:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0ca300 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966951168:192.168.202.146@tcp:37:dbench.0 00010000:00010000:3.0:1775764771.433590:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5c00/0x36f9bdc9e421c267 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8bc4 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1775764771.433605:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5c00/0x36f9bdc9e421c267 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8bc4 expref: -99 pid: 19853 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:3.0:1775764771.433923:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.433929:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.434471:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:3.0:1775764771.434479:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.434485:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:3.0:1775764771.434488:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.434492:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:3.0:1775764771.434495:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.434498:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=ACCESS, flags=17 00000080:00200000:3.0:1775764771.434501:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2e1:0x0](ffff8f9952d91148), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.434512:0:19724:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=FASTENER.MDB, dir=[0x200000401:0x2e1:0x0](ffff8f9952d91148), flags=1 00000080:00200000:3.0:1775764771.434516:0:19724:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=FASTENER.MDB, dir=[0x200000401:0x2e1:0x0](ffff8f9952d91148), intent=getattr 00000002:00010000:3.0:1775764771.434539:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: FASTENER.MDB,[0x0:0x0:0x0]) in obj [0x200000401:0x2e1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764771.434590:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c5cb lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2e1:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.434599:0:19724:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c5cb lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2e1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.434681:0:19724:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c5cb lrc: 3/1,0 mode: --/CR res: [0x200000401:0x2e1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19724 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764771.434705:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970680e00 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966951552:192.168.202.146@tcp:101:dbench.0 00000100:00100000:3.0:1775764771.436050:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413e40 going to sleep for 11 seconds 00000100:00100000:3.0:1775764771.443028:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970680e00 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966951552:192.168.202.146@tcp:101:dbench.0 00010000:00010000:3.0:1775764771.443053:0:19724:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c5cb lrc: 4/1,0 mode: --/CR res: [0x200000401:0x2e1:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa3d4 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.443068:0:19724:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c5cb lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2e3:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa3d4 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.443080:0:19724:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c5cb lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2e3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa3d4 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.443092:0:19724:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c5cb lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2e3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa3d4 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.443102:0:19724:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c5cb lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2e3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa3d4 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.443114:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c5cb lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2e3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa3d4 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.443121:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c5cb lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2e3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa3d4 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.443128:0:19724:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c5cb lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2e3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa3d4 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764771.443138:0:19724:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970680e00 x1862022966951552/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/928 e 0 to 0 dl 1775764787 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'dbench.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1775764771.443148:0:19724:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c5cb lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2e3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa3d4 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:3.0:1775764771.443161:0:19724:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c5cb lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2e3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa3d4 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.443178:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99400/0x36f9bdc9e4216de2 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2e3:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e75d1 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.443189:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99400/0x36f9bdc9e4216de2 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2e3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e75d1 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.443197:0:19724:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c5cb lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2e3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa3d4 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.443203:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c5cb lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2e3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x2aac6a50e5fa3d4 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.443210:0:19724:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c5cb lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x2aac6a50e5fa3d4 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764771.443234:0:19724:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000080:00200000:3.0:1775764771.443241:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2e1:0x0] suppgids 0 -1: rc 0 00000080:00200000:3.0:1775764771.443330:0:19724:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x2e3:0x0](ffff8f9952e4cb08): rc = 0 00000080:00200000:3.0:1775764771.443350:0:19724:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x2e3:0x0]: layout version change: 3 -> 3 00000080:00010000:3.0:1775764771.443360:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2e3:0x0](ffff8f9952e4cb08) for lock 0x36f9bdc9e4216de2 00000080:00010000:3.0:1775764771.443373:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2e3:0x0](ffff8f9952e4cb08) 00000080:00010000:3.0:1775764771.443376:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e4216de2 from it ffffafbac1c5bb20 00010000:00010000:3.0:1775764771.443379:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99400/0x36f9bdc9e4216de2 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2e3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5e75d1 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.443401:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99400/0x36f9bdc9e4216de2 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e75d1 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200008:3.0:1775764771.443453:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file FASTENER.MDB:[0x200000401:0x2e3:0x0](ffff8f9952e4cb08), request_mask 2047, flags 0, foreign 0 00010000:00010000:1.0:1775764771.443981:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764771.443988:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c5cb lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5fa3d4 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.443997:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99435d4800 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:1.0:1775764771.444001:0:19801:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c5cb lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e3:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5fa3d4 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.444008:0:19801:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c5cb lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e3:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5fa3d4 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.444016:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c5cb lrc: 3/0,0 mode: PR/PR res: [0x200000401:0x2e3:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fa3d4 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.444033:0:19801:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99400/0x36f9bdc9e4216de2 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e75d1 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764771.444075:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c5cb lrc: 2/0,0 mode: --/PR res: [0x200000401:0x2e3:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fa3d4 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.444082:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764771.444107:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c5cb lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2e3:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fa3d4 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.444114:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c5cb lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2e3:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fa3d4 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.444128:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764771.444130:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764771.444132:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764771.444139:0:2380:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1775764771.444156:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970b53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966951936:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764771.446604:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970b53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966951936:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764771.455210:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efded80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966952192:192.168.202.146@tcp:36:dbench.0 00000080:00200000:3.0:1775764771.455287:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.455294:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.455301:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:3.0:1775764771.455307:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.455311:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:3.0:1775764771.455314:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.455317:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:3.0:1775764771.455320:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.455323:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:3.0:1775764771.455325:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.455339:0:19723:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=ENROLL.Y03, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), flags=1 00000080:00200000:3.0:1775764771.455343:0:19723:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=ENROLL.Y03, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), intent=getattr 00010000:00010000:3.0:1775764771.455365:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98200/0x36f9bdc9e421c55b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2dc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9ff9 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:3.0:1775764771.455381:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c55b from it ffffafbac1c7b900 00010000:00010000:3.0:1775764771.455385:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98200/0x36f9bdc9e421c55b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2dc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f9ff9 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764771.455394:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98200/0x36f9bdc9e421c55b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2dc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f9ff9 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:3.0:1775764771.455424:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file ENROLL.Y03:[0x200000401:0x2dc:0x0](ffff8f9952d62a08), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764771.455431:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2dc:0x0](ffff8f9952d62a08),name=ENROLL.Y03 00000002:00010000:3.0:1775764771.455449:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2dc:0x0]) in obj [0x200000401:0x2bb:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764771.455461:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925c00/0x36f9bdc9e421bfc7 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2dc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f83eb expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.455473:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925c00/0x36f9bdc9e421bfc7 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2dc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f83eb expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764771.455480:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2bb:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764771.455485:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2dc:0x0](ffff8f9952d62a08) 00000080:00010000:3.0:1775764771.455488:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2dc:0x0](ffff8f9952d62a08) for lock 0x36f9bdc9e421bfc7 00000080:00010000:3.0:1775764771.455493:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421bfc7 from it ffffafbac1c7bcb8 00010000:00010000:3.0:1775764771.455495:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925c00/0x36f9bdc9e421bfc7 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2dc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f83eb expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.455501:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925c00/0x36f9bdc9e421bfc7 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2dc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f83eb expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764771.455548:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2dc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764771.455557:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2dc:0x0] 00000008:00010000:3.0:1775764771.455582:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1775764771.455585:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d55b0, osc lock ffff8f9966ace4c0, flags 0x400001002 00010000:00010000:3.0:1775764771.455602:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000b908cb64/0x36f9bdc9e41cafa8 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x64:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef303 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 65 lru_type: 1 00010000:00010000:3.0:1775764771.455862:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000b908cb64/0x36f9bdc9e41cafa8 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x64:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef303 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 65 lru_type: 1 00000020:00200000:3.0:1775764771.455876:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1775764771.455880:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000b908cb64/0x36f9bdc9e41cafa8 lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x64:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ef303 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 65 lru_type: 1 00010000:00010000:3.0:1775764771.455887:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000b908cb64/0x36f9bdc9e41cafa8 lrc: 4/0,0 mode: PW/PW res: [0x280000403:0x64:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef303 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 66 lru_type: 1 00000020:00200000:3.0:1775764771.455898:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1775764771.455907:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2dc:0x0] updating i_size 260096 i_blocks 1 00000020:00010000:3.0:1775764771.455911:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef52fe8 (ffff8f9970314518/0) at cl_lock_release():245 00000020:00010000:3.0:1775764771.455914:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef52fe8 (ffff8f9970314518/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.455917:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d55b0 (ffff8f9970314f38/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.455920:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef52fe8 (ffff8f9970314518/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764771.455923:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d55b0 (ffff8f9970314518/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764771.456020:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2dc:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.456042:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.456046:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.456049:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:3.0:1775764771.456052:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.456055:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=16 00000080:00200000:3.0:1775764771.456058:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.456060:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:3.0:1775764771.456062:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.456064:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=16 00000080:00200000:3.0:1775764771.456066:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.456070:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=ENROLL.Y03, flags=0 00000080:00200000:3.0:1775764771.456073:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 3 00000080:00200000:3.0:1775764771.456077:0:19723:0:(namei.c:2395:ll_unlink()) VFS Op:name=ENROLL.Y03, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08) 00010000:00010000:3.0:1775764771.456111:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98200/0x36f9bdc9e421c55b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2dc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f9ff9 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764771.456132:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2dc:0x0]: losing layout lock 00010000:00010000:3.0:1775764771.456153:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925c00/0x36f9bdc9e421bfc7 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2dc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f83eb expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764771.456168:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2dc:0x0]: losing layout lock 00010000:00010000:3.0:1775764771.456200:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98200/0x36f9bdc9e421c55b lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2dc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f9ff9 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764771.456208:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925c00/0x36f9bdc9e421bfc7 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2dc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f83eb expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764771.456213:0:19723:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1775764771.456215:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a98200/0x36f9bdc9e421c55b lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2dc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f9ff9 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764771.456226:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925c00/0x36f9bdc9e421bfc7 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2dc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f83eb expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000020:00100000:3.0:1775764771.456245:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1775764771.456255:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970680e00 x1862022966952448/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764771.456276:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970680e00 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966952448:192.168.202.146@tcp:36:dbench.0 00000100:00100000:3.0:1775764771.456369:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413540 going to sleep for 11 seconds 00000080:00200000:3.0:1775764771.458293:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2e3:0x0](ffff8f9952e4cb08),name=FASTENER.MDB 00000002:00010000:3.0:1775764771.458319:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2e3:0x0]) in obj [0x200000401:0x2e1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764771.458334:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99400/0x36f9bdc9e4216de2 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2e3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e75d1 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764771.458349:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99400/0x36f9bdc9e4216de2 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2e3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e75d1 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764771.458356:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2e1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764771.458363:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2e3:0x0](ffff8f9952e4cb08) 00000080:00010000:3.0:1775764771.458366:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2e3:0x0](ffff8f9952e4cb08) for lock 0x36f9bdc9e4216de2 00000080:00010000:3.0:1775764771.458372:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e4216de2 from it ffffafbac1c5bcb8 00010000:00010000:3.0:1775764771.458376:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99400/0x36f9bdc9e4216de2 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2e3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5e75d1 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764771.458384:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a99400/0x36f9bdc9e4216de2 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5e75d1 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1775764771.458432:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2e3:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764771.458441:0:19724:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2e3:0x0] 00000008:00010000:3.0:1775764771.458464:0:19724:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1775764771.458467:0:19724:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d5bb0, osc lock ffff8f9966ace4c0, flags 0x400001002 00010000:00010000:3.0:1775764771.458484:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000053f6e598/0x36f9bdc9e41cb08f lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x67:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef54f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 32 lru_type: 0 00010000:00010000:3.0:1775764771.458495:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000053f6e598/0x36f9bdc9e41cb08f lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x67:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef54f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 32 lru_type: 0 00000020:00200000:3.0:1775764771.458506:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1775764771.458510:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000053f6e598/0x36f9bdc9e41cb08f lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x67:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ef54f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 32 lru_type: 0 00010000:00010000:3.0:1775764771.458523:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000053f6e598/0x36f9bdc9e41cb08f lrc: 4/0,0 mode: PW/PW res: [0x280000403:0x67:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef54f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 33 lru_type: 1 00000020:00200000:3.0:1775764771.458534:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1775764771.458546:0:19724:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2e3:0x0] updating i_size 260096 i_blocks 515 00000020:00010000:3.0:1775764771.458550:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef52fe8 (ffff8f9970314518/0) at cl_lock_release():245 00000020:00010000:3.0:1775764771.458553:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef52fe8 (ffff8f9970314518/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.458556:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d5bb0 (ffff8f9970314f38/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.458560:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef52fe8 (ffff8f9970314518/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764771.458563:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d5bb0 (ffff8f9970314518/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764771.458588:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2e3:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.459101:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.459108:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.459113:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:3.0:1775764771.459119:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.459122:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=16 00000080:00200000:3.0:1775764771.459125:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.459128:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:3.0:1775764771.459131:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.459135:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=ACCESS, flags=16 00000080:00200000:3.0:1775764771.459137:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2e1:0x0](ffff8f9952d91148), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.459143:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=FASTENER.MDB, flags=0 00000100:00100000:3.0:1775764771.467922:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970680e00 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966952448:192.168.202.146@tcp:36:dbench.0 00000080:00200000:3.0:1775764771.468039:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.468047:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.468056:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:3.0:1775764771.468062:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.468066:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:3.0:1775764771.468069:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.468073:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:3.0:1775764771.468076:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.468079:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:3.0:1775764771.468082:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.468097:0:19723:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=STUDENTS.FSL, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), flags=1 00000080:00200000:3.0:1775764771.468102:0:19723:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=STUDENTS.FSL, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), intent=getattr 00010000:00010000:3.0:1775764771.468127:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c562 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2e2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fa015 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:3.0:1775764771.468143:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c562 from it ffffafbac1c7b900 00010000:00010000:3.0:1775764771.468148:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c562 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2e2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fa015 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764771.468157:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c562 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fa015 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:3.0:1775764771.468187:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file STUDENTS.FSL:[0x200000401:0x2e2:0x0](ffff8f9952dd5b88), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764771.468193:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2e2:0x0](ffff8f9952dd5b88),name=STUDENTS.FSL 00000002:00010000:3.0:1775764771.468210:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2e2:0x0]) in obj [0x200000401:0x2bb:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764771.468222:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d8800/0x36f9bdc9e421c07d lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2e2:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8614 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.468233:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d8800/0x36f9bdc9e421c07d lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2e2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8614 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764771.468245:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2bb:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764771.468251:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2e2:0x0](ffff8f9952dd5b88) 00000080:00010000:3.0:1775764771.468254:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2e2:0x0](ffff8f9952dd5b88) for lock 0x36f9bdc9e421c07d 00000080:00010000:3.0:1775764771.468259:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c07d from it ffffafbac1c7bcb8 00010000:00010000:3.0:1775764771.468262:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d8800/0x36f9bdc9e421c07d lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2e2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8614 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.468269:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d8800/0x36f9bdc9e421c07d lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8614 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764771.468315:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2e2:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764771.468325:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2e2:0x0] 00000008:00010000:3.0:1775764771.468354:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1775764771.468357:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d5bb0, osc lock ffff8f9966ace4c0, flags 0x400001002 00010000:00010000:3.0:1775764771.468375:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000070c338a1/0x36f9bdc9e41cb09d lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x67:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef556 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 28 lru_type: 1 00010000:00010000:3.0:1775764771.468398:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000070c338a1/0x36f9bdc9e41cb09d lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x67:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef556 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 28 lru_type: 1 00000020:00200000:3.0:1775764771.468408:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1775764771.468411:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000070c338a1/0x36f9bdc9e41cb09d lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x67:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ef556 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 28 lru_type: 1 00010000:00010000:3.0:1775764771.468418:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000070c338a1/0x36f9bdc9e41cb09d lrc: 4/0,0 mode: PW/PW res: [0x240000403:0x67:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef556 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 29 lru_type: 1 00000020:00200000:3.0:1775764771.468432:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1775764771.468441:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2e2:0x0] updating i_size 260096 i_blocks 1 00000020:00010000:3.0:1775764771.468444:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef52fe8 (ffff8f9970314518/0) at cl_lock_release():245 00000020:00010000:3.0:1775764771.468447:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef52fe8 (ffff8f9970314518/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.468450:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d5bb0 (ffff8f9970314f38/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.468454:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef52fe8 (ffff8f9970314518/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764771.468456:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d5bb0 (ffff8f9970314518/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764771.468484:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2e2:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.468504:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.468507:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.468510:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:3.0:1775764771.468514:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.468516:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=16 00000080:00200000:3.0:1775764771.468518:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.468521:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:3.0:1775764771.468523:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.468525:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=16 00000080:00200000:3.0:1775764771.468527:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.468532:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=STUDENTS.FSL, flags=0 00000080:00200000:3.0:1775764771.468535:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 3 00000080:00200000:3.0:1775764771.468540:0:19723:0:(namei.c:2395:ll_unlink()) VFS Op:name=STUDENTS.FSL, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08) 00010000:00010000:3.0:1775764771.468577:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c562 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e2:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5fa015 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764771.468598:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2e2:0x0]: losing layout lock 00010000:00010000:3.0:1775764771.469110:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d8800/0x36f9bdc9e421c07d lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e2:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f8614 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000100:00100000:3.0:1775764771.473563:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdc380 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966952704:192.168.202.146@tcp:36:dbench.0 00000100:00100000:3.0:1775764771.508644:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdce00 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966953728:192.168.202.146@tcp:36:dbench.0 00000080:00200000:3.0:1775764771.508765:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x274:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1775764771.508780:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764771.508802:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:3.0:1775764771.508804:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:3.0:1775764771.508844:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x274:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.508860:0:19724:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x274:0x0](ffff8f9952da5b88) 00000008:00010000:3.0:1775764771.508903:0:19724:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 1048576 blocks 2051 [cma]time: 1775764050 1775764050 0 by oinfo size 1048576 blocks 2051 [cma]time 1775764050 1775764050 0 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000d94da50a/0x36f9bdc9e421c1cd lrc: 1/0,0 mode: PR/PR res: [0x280000403:0x37:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5f8978 expref: -99 pid: 19850 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764771.509003:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.509010:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.509017:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:3.0:1775764771.509024:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.509027:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=17 00000080:00200000:3.0:1775764771.509030:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.509041:0:19724:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=filler.000, dir=[0x200000401:0x269:0x0](ffff8f9952db63c8), flags=1 00000080:00200000:3.0:1775764771.509046:0:19724:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=filler.000, dir=[0x200000401:0x269:0x0](ffff8f9952db63c8), intent=getattr 00010000:00010000:3.0:1775764771.509062:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bea00/0x36f9bdc9e421c1b8 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f894e expref: -99 pid: 19849 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:3.0:1775764771.509077:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c1b8 from it ffffafbac1c5b900 00010000:00010000:3.0:1775764771.509081:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bea00/0x36f9bdc9e421c1b8 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f894e expref: -99 pid: 19849 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764771.509090:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bea00/0x36f9bdc9e421c1b8 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f894e expref: -99 pid: 19849 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:3.0:1775764771.509121:0:19724:0:(file.c:6226:ll_getattr_dentry()) START file filler.000:[0x200000401:0x26d:0x0](ffff8f9952e65b88), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764771.509127:0:19724:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x26d:0x0](ffff8f9952e65b88),name=filler.000 00000002:00010000:3.0:1775764771.509145:0:19724:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x26d:0x0]) in obj [0x200000401:0x269:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764771.509155:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bea00/0x36f9bdc9e421c1b8 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f894e expref: -99 pid: 19849 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764771.509166:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bea00/0x36f9bdc9e421c1b8 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f894e expref: -99 pid: 19849 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764771.509173:0:19724:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x269:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764771.509178:0:19724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x26d:0x0](ffff8f9952e65b88) 00000080:00010000:3.0:1775764771.509182:0:19724:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x26d:0x0](ffff8f9952e65b88) for lock 0x36f9bdc9e421c1b8 00000080:00010000:3.0:1775764771.509187:0:19724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c1b8 from it ffffafbac1c5bcb8 00010000:00010000:3.0:1775764771.509190:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bea00/0x36f9bdc9e421c1b8 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f894e expref: -99 pid: 19849 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764771.509196:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bea00/0x36f9bdc9e421c1b8 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f894e expref: -99 pid: 19849 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1775764771.509223:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x26d:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764771.509231:0:19724:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x26d:0x0] 00000008:00010000:3.0:1775764771.509255:0:19724:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1775764771.509259:0:19724:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d5bb0, osc lock ffff8f9966ace4c0, flags 0x400001002 00010000:00010000:3.0:1775764771.509277:0:19724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000066a2ef3d/0x36f9bdc9e421c1db lrc: 2/1,0 mode: PR/PR res: [0x240000403:0x34:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f898d expref: -99 pid: 19850 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.509287:0:19724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000066a2ef3d/0x36f9bdc9e421c1db lrc: 2/1,0 mode: PR/PR res: [0x240000403:0x34:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f898d expref: -99 pid: 19850 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:3.0:1775764771.509302:0:19724:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 1048576, setting rss=1048576; kms=1048576, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000066a2ef3d/0x36f9bdc9e421c1db lrc: 4/1,0 mode: PR/PR res: [0x240000403:0x34:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f898d expref: -99 pid: 19850 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:3.0:1775764771.509313:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1775764771.509316:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000066a2ef3d/0x36f9bdc9e421c1db lrc: 5/1,0 mode: PR/PR res: [0x240000403:0x34:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5f898d expref: -99 pid: 19850 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.509324:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000066a2ef3d/0x36f9bdc9e421c1db lrc: 4/0,0 mode: PR/PR res: [0x240000403:0x34:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5f898d expref: -99 pid: 19850 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:3.0:1775764771.509333:0:19724:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1775764771.509343:0:19724:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x26d:0x0] updating i_size 1048576 i_blocks 2051 00000020:00010000:3.0:1775764771.509347:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef52fe8 (ffff8f9970314518/0) at cl_lock_release():245 00000020:00010000:3.0:1775764771.509350:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef52fe8 (ffff8f9970314518/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.509354:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d5bb0 (ffff8f9970314f38/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.509357:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef52fe8 (ffff8f9970314518/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764771.509360:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d5bb0 (ffff8f9970314518/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764771.509374:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x26d:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.509393:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.509397:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.509400:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:3.0:1775764771.509403:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.509406:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=16 00000080:00200000:3.0:1775764771.509408:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.509413:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=filler.000, flags=0 00000080:00200000:3.0:1775764771.509416:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 3 00000080:00200000:3.0:1775764771.509421:0:19724:0:(namei.c:2395:ll_unlink()) VFS Op:name=filler.000, dir=[0x200000401:0x269:0x0](ffff8f9952db63c8) 00010000:00010000:3.0:1775764771.509459:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bea00/0x36f9bdc9e421c1b8 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f894e expref: -99 pid: 19849 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1775764771.509484:0:19724:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x26d:0x0]: losing layout lock 00010000:00010000:3.0:1775764771.509524:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bea00/0x36f9bdc9e421c1b8 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x26d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f894e expref: -99 pid: 19849 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1775764771.509532:0:19724:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764771.509534:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668bea00/0x36f9bdc9e421c1b8 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x26d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f894e expref: -99 pid: 19849 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000020:00100000:3.0:1775764771.509558:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 2 is allocated opc 36, max 7 00000100:00100000:3.0:1775764771.509567:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970680380 x1862022966954240/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764771.509591:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970680380 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966954240:192.168.202.146@tcp:36:dbench.0 00000100:00100000:3.0:1775764771.509689:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413540 going to sleep for 11 seconds 00000100:00100000:3.0:1775764771.514707:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970680380 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966954240:192.168.202.146@tcp:36:dbench.0 00000080:00200000:3.0:1775764771.514823:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x26d:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1775764771.514837:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764771.514860:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:3.0:1775764771.514861:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:3.0:1775764771.514905:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x26d:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.514922:0:19724:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x26d:0x0](ffff8f9952e65b88) 00000008:00010000:3.0:1775764771.515134:0:19724:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 1048576 blocks 2051 [cma]time: 1775764049 1775764049 0 by oinfo size 1048576 blocks 2051 [cma]time 1775764049 1775764049 0 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000066a2ef3d/0x36f9bdc9e421c1db lrc: 1/0,0 mode: PR/PR res: [0x240000403:0x34:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5f898d expref: -99 pid: 19850 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764771.515211:0:19724:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8) pos/size18446744073709551614/7168 32bit_api 0 00000080:00200008:3.0:1775764771.515234:0:19724:0:(file.c:447:ll_file_release()) START file client1:[0x200000401:0x269:0x0](ffff8f9952db63c8), flags 304000 00000002:00100000:3.0:1775764771.515306:0:19724:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f994f2e5f80 x1862022966906880/t4294997340(4294997340) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764786 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000020:00100000:3.0:1775764771.515318:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 2 is allocated opc 35, max 7 00000100:00100000:3.0:1775764771.515322:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970682a00 x1862022966954496/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:3.0:1775764771.515345:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970682a00 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966954496:192.168.202.146@tcp:35:dbench.0 00000100:00100000:3.0:1775764771.515437:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413540 going to sleep for 11 seconds 00000100:00100000:3.0:1775764771.534951:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdf480 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966954752:192.168.202.146@tcp:36:dbench.0 00000080:00200000:3.0:1775764771.535037:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xece:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1775764771.535049:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764771.535066:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:3.0:1775764771.535068:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000008:00010000:3.0:1775764771.535085:0:19723:0:(osc_cache.c:1167:osc_extent_make_ready()) ### extent: 0000000003815a0b ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000fb0a8ec1/0x36f9bdc9e421b2df lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x65b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f5cb5 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00000008:00010000:3.0:1775764771.535116:0:19723:0:(osc_cache.c:901:osc_extent_finish()) ### extent: 0000000003815a0b ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000fb0a8ec1/0x36f9bdc9e421b2df lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x65b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f5cb5 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00000008:00010000:3.0:1775764771.535141:0:19723:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 0000000003815a0b ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000fb0a8ec1/0x36f9bdc9e421b2df lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x65b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f5cb5 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00000080:00200000:3.0:1775764771.535175:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xece:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.535260:0:19723:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0xece:0x0](ffff8f9952eb3a88) 00000008:00010000:3.0:1775764771.535304:0:19723:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 19584 blocks 1 [cma]time 1775764767 1775764767 0 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000fb0a8ec1/0x36f9bdc9e421b2df lrc: 1/0,0 mode: PW/PW res: [0x280000403:0x65b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f5cb5 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00000080:00200000:3.0:1775764771.535392:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.535398:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.535404:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:3.0:1775764771.535410:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.535413:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:3.0:1775764771.535416:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.535419:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:3.0:1775764771.535421:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.535424:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:3.0:1775764771.535426:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.535439:0:19723:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=STUDENTS.MB, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), flags=1 00000080:00200000:3.0:1775764771.535443:0:19723:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=STUDENTS.MB, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), intent=getattr 00010000:00010000:3.0:1775764771.535460:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebe00/0x36f9bdc9e421c5af lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2d4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fa17a expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:3.0:1775764771.535475:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c5af from it ffffafbac1c7b900 00010000:00010000:3.0:1775764771.535479:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebe00/0x36f9bdc9e421c5af lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2d4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fa17a expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764771.535488:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebe00/0x36f9bdc9e421c5af lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fa17a expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:3.0:1775764771.535514:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file STUDENTS.MB:[0x200000401:0x2d4:0x0](ffff8f9952d95b88), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764771.535520:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2d4:0x0](ffff8f9952d95b88),name=STUDENTS.MB 00000002:00010000:3.0:1775764771.535538:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2d4:0x0]) in obj [0x200000401:0x2bb:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764771.535548:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49e00/0x36f9bdc9e421c01b lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2d4:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f84e7 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.535559:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49e00/0x36f9bdc9e421c01b lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2d4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f84e7 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764771.535566:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2bb:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764771.535572:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2d4:0x0](ffff8f9952d95b88) 00000080:00010000:3.0:1775764771.535576:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2d4:0x0](ffff8f9952d95b88) for lock 0x36f9bdc9e421c01b 00000080:00010000:3.0:1775764771.535580:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c01b from it ffffafbac1c7bcb8 00010000:00010000:3.0:1775764771.535583:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49e00/0x36f9bdc9e421c01b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2d4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f84e7 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764771.535590:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49e00/0x36f9bdc9e421c01b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f84e7 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764771.535768:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2d4:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1775764771.535775:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2d4:0x0] 00000008:00010000:3.0:1775764771.535798:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1775764771.535802:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99525d5bb0, osc lock ffff8f9966ace4c0, flags 0x400001002 00010000:00010000:3.0:1775764771.535817:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f444aa51/0x36f9bdc9e41cae97 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x60:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef055 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 138 lru_type: 1 00010000:00010000:3.0:1775764771.535832:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f444aa51/0x36f9bdc9e41cae97 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x60:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef055 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 138 lru_type: 1 00000020:00200000:3.0:1775764771.535844:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1775764771.535847:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f444aa51/0x36f9bdc9e41cae97 lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x60:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ef055 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 138 lru_type: 1 00010000:00010000:3.0:1775764771.535856:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f444aa51/0x36f9bdc9e41cae97 lrc: 4/0,0 mode: PW/PW res: [0x280000403:0x60:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef055 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 139 lru_type: 1 00000020:00200000:3.0:1775764771.535866:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1775764771.535875:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2d4:0x0] updating i_size 260096 i_blocks 1 00000020:00010000:3.0:1775764771.535879:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f994ef52fe8 (ffff8f9970314518/0) at cl_lock_release():245 00000020:00010000:3.0:1775764771.535882:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994ef52fe8 (ffff8f9970314518/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.535886:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99525d5bb0 (ffff8f9970314f38/1) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.535890:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994ef52fe8 (ffff8f9970314518/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764771.535892:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99525d5bb0 (ffff8f9970314518/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764771.535907:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2d4:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.535926:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.535931:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.535944:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:3.0:1775764771.535948:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764771.535951:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=16 00000080:00200000:3.0:1775764771.535953:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.535955:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:3.0:1775764771.535957:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.535960:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=16 00000080:00200000:3.0:1775764771.535962:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764771.535966:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=STUDENTS.MB, flags=0 00000080:00200000:3.0:1775764771.535969:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 3 00000080:00200000:3.0:1775764771.535973:0:19723:0:(namei.c:2395:ll_unlink()) VFS Op:name=STUDENTS.MB, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08) 00010000:00010000:3.0:1775764771.536011:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebe00/0x36f9bdc9e421c5af lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d4:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5fa17a expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764771.536030:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2d4:0x0]: losing layout lock 00010000:00010000:3.0:1775764771.536047:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49e00/0x36f9bdc9e421c01b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d4:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f84e7 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764771.536059:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2d4:0x0]: losing layout lock 00010000:00010000:3.0:1775764771.536088:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebe00/0x36f9bdc9e421c5af lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2d4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fa17a expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764771.536095:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49e00/0x36f9bdc9e421c01b lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2d4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f84e7 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764771.536100:0:19723:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1775764771.536102:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebe00/0x36f9bdc9e421c5af lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2d4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fa17a expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764771.536112:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d49e00/0x36f9bdc9e421c01b lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2d4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f84e7 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000020:00100000:3.0:1775764771.536132:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1775764771.536141:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970682a00 x1862022966955136/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764771.536169:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970682a00 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966955136:192.168.202.146@tcp:36:dbench.0 00000100:00100000:3.0:1775764771.536248:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413d80 going to sleep for 11 seconds 00000008:00010000:3.0:1775764771.541220:0:2381:0:(osc_cache.c:901:osc_extent_finish()) ### no dlm lock: extent: ffff8f9961124d80 00000020:00200000:3.0:1775764771.541234:0:2381:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:3.0:1775764771.541255:0:2381:0:(osc_cache.c:496:__osc_extent_remove()) ### no dlm lock: extent: ffff8f9961124d80 00000100:00100000:3.0:1775764771.541279:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efde680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966954880:192.168.202.146@tcp:3:dbench.0 00000020:00010000:0.0:1775764771.541478:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970942e48 (ffff8f994ef58008/0) at cl_lock_release():245 00000020:00010000:0.0:1775764771.541483:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970942e48 (ffff8f994ef58008/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764771.541488:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994166f430 (ffff8f994ef586c8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764771.541498:0:19724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 4/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:0.0:1775764771.541513:0:19724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 3/0,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00000020:00010000:0.0:1775764771.541527:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970942e48 (ffff8f994ef58008/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764771.541531:0:19724:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994166f430 (ffff8f994ef58008/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764771.541567:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x266:0x0] ignore/verify layout 0/0, layout version 4 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764771.541674:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.541680:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.541690:0:19724:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=0 00000080:00200000:0.0:1775764771.541697:0:19724:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 3 00000080:00200000:0.0:1775764771.541702:0:19724:0:(namei.c:2305:ll_rmdir()) VFS Op:name=clients, dir=[0x200000401:0x261:0x0](ffff8f9952e61988) 00010000:00010000:0.0:1775764771.541730:0:19724:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971070600/0x36f9bdc9e421370f lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x268:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5dcb38 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764771.541815:0:19724:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971070600/0x36f9bdc9e421370f lrc: 1/0,0 mode: --/PR res: [0x200000401:0x268:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5dcb38 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764771.541823:0:19724:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764771.541825:0:19724:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971070600/0x36f9bdc9e421370f lrc: 0/0,0 mode: --/PR res: [0x200000401:0x268:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5dcb38 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000020:00100000:0.0:1775764771.541852:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 2 is allocated opc 36, max 7 00000100:00100000:0.0:1775764771.541861:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f0c8000 x1862022966955392/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 488/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764771.541884:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c8000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966955392:192.168.202.146@tcp:36:dbench.0 00000100:00100000:0.0:1775764771.542643:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f3c0 going to sleep for 11 seconds 00000100:00100000:3.0:1775764771.545896:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970682a00 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966955136:192.168.202.146@tcp:36:dbench.0 00000080:00200000:0.0:1775764771.546901:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.546911:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000100:00100000:3.0:1775764771.548528:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966955520:192.168.202.146@tcp:3:dbench.0 00000080:00200000:0.0:1775764771.549722:0:19723:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=clients, dir=[0x200000401:0x261:0x0](ffff8f9952e61988), flags=17 00000080:00200000:0.0:1775764771.549733:0:19723:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=clients, dir=[0x200000401:0x261:0x0](ffff8f9952e61988), intent=0 00000002:00010000:0.0:1775764771.549784:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: clients,[0x0:0x0:0x0]) in obj [0x200000401:0x261:0x0], intent: lookup flags 0 00010000:00010000:0.0:1775764771.549841:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c5d2 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.549855:0:19723:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c5d2 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.549865:0:19723:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c5d2 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 19723 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764771.549886:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c8000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966955648:192.168.202.146@tcp:101:dbench.0 00000100:00100000:0.0:1775764771.549971:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f3c0 going to sleep for 11 seconds 00000008:00010000:3.0:1775764771.558359:0:2381:0:(osc_cache.c:901:osc_extent_finish()) ### no dlm lock: extent: ffff8f99611246c0 00000020:00200000:3.0:1775764771.558373:0:2381:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:3.0:1775764771.558400:0:2381:0:(osc_cache.c:496:__osc_extent_remove()) ### no dlm lock: extent: ffff8f99611246c0 00000100:00100000:3.0:1775764771.558428:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966955520:192.168.202.146@tcp:3:dbench.0 00000100:00100000:3.0:1775764771.559263:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966955776:192.168.202.146@tcp:3:dbench.0 00000100:00100000:0.0:1775764771.560952:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c8000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966955648:192.168.202.146@tcp:101:dbench.0 00010000:00010000:0.0:1775764771.560974:0:19723:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c5d2 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa524 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.561008:0:19723:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c5d2 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x268:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa524 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.561016:0:19723:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c5d2 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x268:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa524 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.561028:0:19723:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c5d2 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x268:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa524 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.561039:0:19723:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c5d2 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x268:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa524 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.561050:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c5d2 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x268:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa524 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.561064:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c5d2 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x268:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa524 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.561072:0:19723:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c5d2 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x268:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa524 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764771.561083:0:19723:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=b, status=0 req@ffff8f994f0c8000 x1862022966955648/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/584 e 0 to 0 dl 1775764787 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'dbench.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1775764771.561096:0:19723:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c5d2 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x268:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa524 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.561105:0:19723:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/616 (0 0) 00000080:00200000:0.0:1775764771.561112:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x261:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1775764771.561132:0:19723:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x268:0x0], updating i_size 7168 00000080:00200000:0.0:1775764771.561135:0:19723:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x268:0x0](ffff8f9952e63a88): rc = 0 00000080:00010000:0.0:1775764771.561141:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x268:0x0](ffff8f9952e63a88) for lock 0x36f9bdc9e421c5d2 00000080:00010000:0.0:1775764771.561158:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x268:0x0](ffff8f9952e63a88) 00000080:00010000:0.0:1775764771.561162:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c5d2 from it ffffafbac1c7ba28 00010000:00010000:0.0:1775764771.561165:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c5d2 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x268:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa524 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.561174:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c5d2 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x268:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa524 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764771.561218:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.561228:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:0.0:1775764771.561233:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.561236:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:0.0:1775764771.561239:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.561242:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:0.0:1775764771.561244:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.561258:0:19723:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=STUDENTS.YG1, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), flags=1 00000080:00200000:0.0:1775764771.561261:0:19723:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=STUDENTS.YG1, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), intent=getattr 00010000:00010000:0.0:1775764771.561275:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c5b6 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2d0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fa196 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:0.0:1775764771.561285:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c5b6 from it ffffafbac1c7b900 00010000:00010000:0.0:1775764771.561289:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c5b6 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2d0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fa196 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764771.561295:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c5b6 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fa196 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:0.0:1775764771.561316:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file STUDENTS.YG1:[0x200000401:0x2d0:0x0](ffff8f9952d94b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764771.561322:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2d0:0x0](ffff8f9952d94b08),name=STUDENTS.YG1 00000002:00010000:0.0:1775764771.561339:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2d0:0x0]) in obj [0x200000401:0x2bb:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764771.561348:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8de00/0x36f9bdc9e421beee lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2d0:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8160 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764771.561358:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8de00/0x36f9bdc9e421beee lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2d0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8160 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764771.561364:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2bb:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764771.561368:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2d0:0x0](ffff8f9952d94b08) 00000080:00010000:0.0:1775764771.561372:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2d0:0x0](ffff8f9952d94b08) for lock 0x36f9bdc9e421beee 00000080:00010000:0.0:1775764771.561375:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421beee from it ffffafbac1c7bcb8 00010000:00010000:0.0:1775764771.561378:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8de00/0x36f9bdc9e421beee lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2d0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8160 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764771.561401:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8de00/0x36f9bdc9e421beee lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8160 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764771.561447:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2d0:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1775764771.561458:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2d0:0x0] 00000008:00010000:0.0:1775764771.561485:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1775764771.561489:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0df830, osc lock ffff8f996686ee40, flags 0x400001002 00010000:00010000:0.0:1775764771.561509:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c8469d80/0x36f9bdc9e41cae0b lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x5e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eef05 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 38 lru_type: 1 00010000:00010000:0.0:1775764771.561521:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c8469d80/0x36f9bdc9e41cae0b lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x5e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eef05 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 38 lru_type: 1 00000020:00200000:0.0:1775764771.561530:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1775764771.561533:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c8469d80/0x36f9bdc9e41cae0b lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x5e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4eef05 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 38 lru_type: 1 00010000:00010000:0.0:1775764771.561540:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c8469d80/0x36f9bdc9e41cae0b lrc: 4/0,0 mode: PW/PW res: [0x280000403:0x5e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eef05 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 39 lru_type: 1 00000020:00200000:0.0:1775764771.561548:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1775764771.561556:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2d0:0x0] updating i_size 260096 i_blocks 1 00000020:00010000:0.0:1775764771.561559:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970b8b908 (ffff8f9966dc2f38/0) at cl_lock_release():245 00000020:00010000:0.0:1775764771.561562:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970b8b908 (ffff8f9966dc2f38/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764771.561565:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0df830 (ffff8f9966dc2998/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764771.561568:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970b8b908 (ffff8f9966dc2f38/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764771.561571:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0df830 (ffff8f9966dc2f38/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764771.561593:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2d0:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764771.561630:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.561634:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.561637:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:0.0:1775764771.561640:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.561643:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=16 00000080:00200000:0.0:1775764771.561645:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.561648:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:0.0:1775764771.561650:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.561653:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=16 00000080:00200000:0.0:1775764771.561655:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.561659:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=STUDENTS.YG1, flags=0 00000080:00200000:0.0:1775764771.561663:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 3 00000080:00200000:0.0:1775764771.561667:0:19723:0:(namei.c:2395:ll_unlink()) VFS Op:name=STUDENTS.YG1, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08) 00010000:00010000:0.0:1775764771.561706:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c5b6 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d0:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5fa196 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1775764771.561727:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2d0:0x0]: losing layout lock 00010000:00010000:0.0:1775764771.561920:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8de00/0x36f9bdc9e421beee lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2d0:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f8160 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764771.561950:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2d0:0x0]: losing layout lock 00010000:00010000:0.0:1775764771.561981:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c5b6 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2d0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fa196 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764771.561993:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8de00/0x36f9bdc9e421beee lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2d0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f8160 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764771.561998:0:19723:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1775764771.562001:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c5b6 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2d0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fa196 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764771.562014:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994ed8de00/0x36f9bdc9e421beee lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2d0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f8160 expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000020:00100000:0.0:1775764771.562032:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1775764771.562039:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f0c8000 x1862022966956032/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764771.562062:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c8000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966956032:192.168.202.146@tcp:36:dbench.0 00000100:00100000:0.0:1775764771.562132:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f3c0 going to sleep for 11 seconds 00000008:00010000:3.0:1775764771.565561:0:2382:0:(osc_cache.c:901:osc_extent_finish()) ### no dlm lock: extent: ffff8f9961124b40 00000020:00200000:3.0:1775764771.565577:0:2382:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:3.0:1775764771.565598:0:2382:0:(osc_cache.c:496:__osc_extent_remove()) ### no dlm lock: extent: ffff8f9961124b40 00000100:00100000:3.0:1775764771.565635:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966955776:192.168.202.146@tcp:3:dbench.0 00000100:00100000:0.0:1775764771.573225:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c8000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966956032:192.168.202.146@tcp:36:dbench.0 00000080:00200000:0.0:1775764771.573282:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.573287:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.573293:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:0.0:1775764771.573299:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.573302:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:0.0:1775764771.573305:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.573308:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=17 00000080:00200000:0.0:1775764771.573311:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.573314:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=17 00000080:00200000:0.0:1775764771.573317:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.573330:0:19723:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=ENROLL.X03, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), flags=1 00000080:00200000:0.0:1775764771.573335:0:19723:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=ENROLL.X03, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), intent=getattr 00010000:00010000:0.0:1775764771.573355:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb000/0x36f9bdc9e421c5bd lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2da:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fa1b2 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:0.0:1775764771.573371:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c5bd from it ffffafbac1c7b900 00010000:00010000:0.0:1775764771.573375:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb000/0x36f9bdc9e421c5bd lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2da:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fa1b2 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764771.573384:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb000/0x36f9bdc9e421c5bd lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2da:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fa1b2 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:0.0:1775764771.573425:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file ENROLL.X03:[0x200000401:0x2da:0x0](ffff8f9952d95348), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764771.573431:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2da:0x0](ffff8f9952d95348),name=ENROLL.X03 00000002:00010000:0.0:1775764771.573449:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2da:0x0]) in obj [0x200000401:0x2bb:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764771.573459:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925400/0x36f9bdc9e421bf96 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2da:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f835f expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764771.573468:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925400/0x36f9bdc9e421bf96 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2da:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f835f expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764771.573476:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x2bb:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764771.573482:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2da:0x0](ffff8f9952d95348) 00000080:00010000:0.0:1775764771.573486:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2da:0x0](ffff8f9952d95348) for lock 0x36f9bdc9e421bf96 00000080:00010000:0.0:1775764771.573491:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421bf96 from it ffffafbac1c7bcb8 00010000:00010000:0.0:1775764771.573494:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925400/0x36f9bdc9e421bf96 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2da:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f835f expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764771.573505:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925400/0x36f9bdc9e421bf96 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2da:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f835f expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764771.573541:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2da:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1775764771.573549:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x2da:0x0] 00000008:00010000:0.0:1775764771.573573:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1775764771.573577:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0df830, osc lock ffff8f996686ee40, flags 0x400001002 00010000:00010000:0.0:1775764771.573597:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000000b928e73/0x36f9bdc9e41caf70 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x63:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef24d expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 65 lru_type: 1 00010000:00010000:0.0:1775764771.573608:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000000b928e73/0x36f9bdc9e41caf70 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x63:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef24d expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 65 lru_type: 1 00000020:00200000:0.0:1775764771.573630:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1775764771.573634:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000000b928e73/0x36f9bdc9e41caf70 lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x63:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e4ef24d expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 65 lru_type: 1 00010000:00010000:0.0:1775764771.573642:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000000b928e73/0x36f9bdc9e41caf70 lrc: 4/0,0 mode: PW/PW res: [0x280000403:0x63:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef24d expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 66 lru_type: 1 00000020:00200000:0.0:1775764771.573652:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1775764771.573661:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x2da:0x0] updating i_size 260096 i_blocks 1 00000020:00010000:0.0:1775764771.573664:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970b8b908 (ffff8f9966dc2f38/0) at cl_lock_release():245 00000020:00010000:0.0:1775764771.573667:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970b8b908 (ffff8f9966dc2f38/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764771.573670:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0df830 (ffff8f9966dc2998/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764771.573673:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970b8b908 (ffff8f9966dc2f38/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764771.573677:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0df830 (ffff8f9966dc2f38/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764771.573695:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2da:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764771.573715:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.573719:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.573722:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:0.0:1775764771.573725:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.573727:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=16 00000080:00200000:0.0:1775764771.573729:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.573732:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=16 00000080:00200000:0.0:1775764771.573733:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.573735:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=16 00000080:00200000:0.0:1775764771.573775:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.573780:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=ENROLL.X03, flags=0 00000080:00200000:0.0:1775764771.573784:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 3 00000080:00200000:0.0:1775764771.573789:0:19723:0:(namei.c:2395:ll_unlink()) VFS Op:name=ENROLL.X03, dir=[0x200000401:0x2bb:0x0](ffff8f9952da4b08) 00010000:00010000:0.0:1775764771.573825:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb000/0x36f9bdc9e421c5bd lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2da:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5fa1b2 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1775764771.573847:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2da:0x0]: losing layout lock 00010000:00010000:0.0:1775764771.573868:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925400/0x36f9bdc9e421bf96 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2da:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f835f expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764771.573882:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x2da:0x0]: losing layout lock 00010000:00010000:0.0:1775764771.573911:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb000/0x36f9bdc9e421c5bd lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2da:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fa1b2 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764771.573918:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925400/0x36f9bdc9e421bf96 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2da:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f835f expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764771.573928:0:19723:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1775764771.573931:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb000/0x36f9bdc9e421c5bd lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2da:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fa1b2 expref: -99 pid: 19887 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764771.573943:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925400/0x36f9bdc9e421bf96 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2da:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f835f expref: -99 pid: 19723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000020:00100000:0.0:1775764771.573961:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1775764771.573967:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f0ca300 x1862022966956800/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764771.573981:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0ca300 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966956800:192.168.202.146@tcp:36:dbench.0 00000100:00100000:0.0:1775764771.574018:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f3c0 going to sleep for 11 seconds 00000100:00100000:0.0:1775764771.580204:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0ca300 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966956800:192.168.202.146@tcp:36:dbench.0 00000080:00200000:0.0:1775764771.580256:0:19723:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08) pos/size18446744073709551614/8192 32bit_api 0 00000080:00200008:0.0:1775764771.580276:0:19723:0:(file.c:447:ll_file_release()) START file PARADOX:[0x200000401:0x2bb:0x0](ffff8f9952da4b08), flags 304000 00010000:00010000:0.0:1775764771.580319:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d9c00/0x36f9bdc9e41cab4f lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ee860 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764771.580344:0:19723:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08) pos/size18446744073709551614/7168 32bit_api 0 00000080:00200008:0.0:1775764771.580431:0:19723:0:(file.c:447:ll_file_release()) START file ~dmtmp:[0x200000401:0x26c:0x0](ffff8f9952db6c08), flags 304000 00010000:00010000:0.0:1775764771.580460:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971078400/0x36f9bdc9e41ca50c lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ed7b9 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764771.580495:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.580502:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.580508:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=17 00000080:00200000:0.0:1775764771.580513:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.580516:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=17 00000080:00200000:0.0:1775764771.580519:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.580529:0:19723:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=filler.001, dir=[0x200000401:0x26a:0x0](ffff8f9952da00c8), flags=1 00000080:00200000:0.0:1775764771.580534:0:19723:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=filler.001, dir=[0x200000401:0x26a:0x0](ffff8f9952da00c8), intent=getattr 00010000:00010000:0.0:1775764771.580549:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c1fe lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8a20 expref: -99 pid: 19851 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:0.0:1775764771.580563:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c1fe from it ffffafbac1c7b900 00010000:00010000:0.0:1775764771.580566:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c1fe lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8a20 expref: -99 pid: 19851 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764771.580574:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c1fe lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8a20 expref: -99 pid: 19851 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:0.0:1775764771.580627:0:19723:0:(file.c:6226:ll_getattr_dentry()) START file filler.001:[0x200000401:0x26f:0x0](ffff8f9952e621c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764771.580633:0:19723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x26f:0x0](ffff8f9952e621c8),name=filler.001 00000002:00010000:0.0:1775764771.580650:0:19723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x26f:0x0]) in obj [0x200000401:0x26a:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764771.580658:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c1fe lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26f:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8a20 expref: -99 pid: 19851 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764771.580666:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c1fe lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8a20 expref: -99 pid: 19851 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1775764771.580673:0:19723:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26a:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764771.580679:0:19723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x26f:0x0](ffff8f9952e621c8) 00000080:00010000:0.0:1775764771.580682:0:19723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x26f:0x0](ffff8f9952e621c8) for lock 0x36f9bdc9e421c1fe 00000080:00010000:0.0:1775764771.580688:0:19723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c1fe from it ffffafbac1c7bcb8 00010000:00010000:0.0:1775764771.580691:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c1fe lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8a20 expref: -99 pid: 19851 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764771.580698:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c1fe lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8a20 expref: -99 pid: 19851 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:0.0:1775764771.580786:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x26f:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1775764771.580796:0:19723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x26f:0x0] 00000008:00010000:0.0:1775764771.580834:0:19723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1775764771.580838:0:19723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0df830, osc lock ffff8f996686ee40, flags 0x400001002 00010000:00010000:0.0:1775764771.580855:0:19723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ecd712f3/0x36f9bdc9e421c221 lrc: 2/1,0 mode: PR/PR res: [0x240000403:0x35:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8a3c expref: -99 pid: 19852 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764771.580868:0:19723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ecd712f3/0x36f9bdc9e421c221 lrc: 2/1,0 mode: PR/PR res: [0x240000403:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5f8a3c expref: -99 pid: 19852 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:0.0:1775764771.580881:0:19723:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 1048576, setting rss=1048576; kms=1048576, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ecd712f3/0x36f9bdc9e421c221 lrc: 4/1,0 mode: PR/PR res: [0x240000403:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5f8a3c expref: -99 pid: 19852 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:0.0:1775764771.580892:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1775764771.580896:0:19723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ecd712f3/0x36f9bdc9e421c221 lrc: 5/1,0 mode: PR/PR res: [0x240000403:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5f8a3c expref: -99 pid: 19852 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764771.580904:0:19723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ecd712f3/0x36f9bdc9e421c221 lrc: 4/0,0 mode: PR/PR res: [0x240000403:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5f8a3c expref: -99 pid: 19852 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:0.0:1775764771.580915:0:19723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1775764771.580927:0:19723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x26f:0x0] updating i_size 1048576 i_blocks 2051 00000020:00010000:0.0:1775764771.580931:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970b8b908 (ffff8f9966dc2f38/0) at cl_lock_release():245 00000020:00010000:0.0:1775764771.580935:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970b8b908 (ffff8f9966dc2f38/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764771.580950:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0df830 (ffff8f9966dc2998/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764771.580954:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970b8b908 (ffff8f9966dc2f38/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764771.580957:0:19723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0df830 (ffff8f9966dc2f38/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764771.580980:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x26f:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764771.581002:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.581005:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.581009:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=16 00000080:00200000:0.0:1775764771.581012:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.581015:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=16 00000080:00200000:0.0:1775764771.581017:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764771.581021:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=filler.001, flags=0 00000080:00200000:0.0:1775764771.581025:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 3 00000080:00200000:0.0:1775764771.581029:0:19723:0:(namei.c:2395:ll_unlink()) VFS Op:name=filler.001, dir=[0x200000401:0x26a:0x0](ffff8f9952da00c8) 00010000:00010000:0.0:1775764771.581064:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c1fe lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26f:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f8a20 expref: -99 pid: 19851 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:0.0:1775764771.581087:0:19723:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x26f:0x0]: losing layout lock 00010000:00010000:0.0:1775764771.581127:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c1fe lrc: 1/0,0 mode: --/PR res: [0x200000401:0x26f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f8a20 expref: -99 pid: 19851 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764771.581135:0:19723:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764771.581137:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668be600/0x36f9bdc9e421c1fe lrc: 0/0,0 mode: --/PR res: [0x200000401:0x26f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f8a20 expref: -99 pid: 19851 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000020:00100000:0.0:1775764771.581160:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1775764771.581169:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f0c9500 x1862022966957056/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764771.581190:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c9500 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966957056:192.168.202.146@tcp:36:dbench.0 00000100:00100000:0.0:1775764771.581264:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f3c0 going to sleep for 11 seconds 00000100:00100000:3.0:1775764771.595200:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdf800 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966957312:192.168.202.146@tcp:35:dbench.0 00000002:00100000:3.0:1775764771.595227:0:19724:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f9970c01500 x1862022966894336/t4294997325(4294997325) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/952 e 0 to 0 dl 1775764785 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1775764771.595317:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2d9:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1775764771.595330:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764771.595351:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:3.0:1775764771.595353:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:3.0:1775764771.595399:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2d9:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.595900:0:19724:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2d9:0x0](ffff8f9952dd1148) 00000008:00010000:3.0:1775764771.595945:0:19724:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 1 [cma]time 1775764058 1775764058 0 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000008eb5a6c2/0x36f9bdc9e41caf4d lrc: 1/0,0 mode: PW/PW res: [0x240000403:0x63:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef207 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 58 lru_type: 1 00000080:00200008:3.0:1775764771.596005:0:19724:0:(file.c:447:ll_file_release()) START file ENROLL.Y03:[0x200000401:0x2d7:0x0](ffff8f9952e4f448), flags 100002 00000002:00100000:3.0:1775764771.596056:0:19724:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f99703de680 x1862022966892288/t4294997323(4294997323) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/952 e 0 to 0 dl 1775764785 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000020:00100000:3.0:1775764771.596066:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 2 is allocated opc 35, max 7 00000100:00100000:3.0:1775764771.596070:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970680000 x1862022966957568/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:3.0:1775764771.596090:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970680000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966957568:192.168.202.146@tcp:35:dbench.0 00000100:00100000:3.0:1775764771.596155:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413d80 going to sleep for 11 seconds 00000100:00100000:0.0:1775764771.596854:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c9500 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966957056:192.168.202.146@tcp:36:dbench.0 00000080:00200000:0.0:1775764771.596915:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x26f:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764771.596924:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1775764771.596940:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:0.0:1775764771.596942:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:0.0:1775764771.596970:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x26f:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764771.596981:0:19723:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x26f:0x0](ffff8f9952e621c8) 00000008:00010000:0.0:1775764771.597004:0:19723:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 1048576 blocks 2051 [cma]time: 1775764050 1775764050 0 by oinfo size 1048576 blocks 2051 [cma]time 1775764050 1775764050 0 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ecd712f3/0x36f9bdc9e421c221 lrc: 1/0,0 mode: PR/PR res: [0x240000403:0x35:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5f8a3c expref: -99 pid: 19852 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764771.597057:0:19723:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8) pos/size18446744073709551614/7168 32bit_api 0 00000080:00200008:0.0:1775764771.597073:0:19723:0:(file.c:447:ll_file_release()) START file client0:[0x200000401:0x26a:0x0](ffff8f9952da00c8), flags 304000 00000002:00100000:0.0:1775764771.597112:0:19723:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f99703dc380 x1862022966906624/t4294997339(4294997339) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764786 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1775764771.597120:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1775764771.597124:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f0c9180 x1862022966957824/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1775764771.597136:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c9180 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966957824:192.168.202.146@tcp:35:dbench.0 00000100:00100000:0.0:1775764771.597191:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f3c0 going to sleep for 11 seconds 00000100:00100000:3.0:1775764771.605399:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970680000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966957568:192.168.202.146@tcp:35:dbench.0 00000002:00100000:3.0:1775764771.605430:0:19724:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f99703de680 x1862022966892288/t4294997323(4294997323) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/952 e 0 to 0 dl 1775764785 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1775764771.605507:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2d7:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1775764771.605520:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764771.605541:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:3.0:1775764771.605542:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:3.0:1775764771.605577:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2d7:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.606224:0:19724:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2d7:0x0](ffff8f9952e4f448) 00000008:00010000:3.0:1775764771.606268:0:19724:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 1 [cma]time 1775764058 1775764058 0 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a81a0d9c/0x36f9bdc9e41caf07 lrc: 1/0,0 mode: PW/PW res: [0x240000403:0x62:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef15f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 41 lru_type: 1 00000080:00200008:3.0:1775764771.606334:0:19724:0:(file.c:447:ll_file_release()) START file ENROLL.X03:[0x200000401:0x2d5:0x0](ffff8f9952e2ba88), flags 100002 00000002:00100000:3.0:1775764771.606392:0:19724:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f99463c0000 x1862022966890624/t4294997321(4294997321) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/952 e 0 to 0 dl 1775764785 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000020:00100000:3.0:1775764771.606403:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 2 is allocated opc 35, max 7 00000100:00100000:3.0:1775764771.606408:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970680000 x1862022966958080/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:3.0:1775764771.606429:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970680000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966958080:192.168.202.146@tcp:35:dbench.0 00000100:00100000:3.0:1775764771.606544:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413d80 going to sleep for 11 seconds 00000100:00100000:0.0:1775764771.606801:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c9180 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966957824:192.168.202.146@tcp:35:dbench.0 00000002:00100000:0.0:1775764771.606817:0:19723:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f99703dc380 x1862022966906624/t4294997339(4294997339) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764786 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1775764771.606945:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.606952:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.606960:0:19723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=clients, flags=0 00000080:00200000:0.0:1775764771.606968:0:19723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 3 00000080:00200000:0.0:1775764771.606973:0:19723:0:(namei.c:2305:ll_rmdir()) VFS Op:name=clients, dir=[0x200000401:0x261:0x0](ffff8f9952e61988) 00010000:00010000:0.0:1775764771.606995:0:19723:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c5d2 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x268:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fa524 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764771.607037:0:19723:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c5d2 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x268:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fa524 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764771.607044:0:19723:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764771.607047:0:19723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c5d2 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x268:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fa524 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000020:00100000:0.0:1775764771.607066:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1775764771.607072:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f0c9180 x1862022966958336/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 488/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764771.607084:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c9180 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966958336:192.168.202.146@tcp:36:dbench.0 00000100:00100000:0.0:1775764771.607115:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f3c0 going to sleep for 11 seconds 00000100:00100000:0.0:1775764771.620839:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c9180 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966958336:192.168.202.146@tcp:36:dbench.0 00000100:00100000:3.0:1775764771.624195:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970680000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966958080:192.168.202.146@tcp:35:dbench.0 00000002:00100000:3.0:1775764771.624229:0:19724:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f99463c0000 x1862022966890624/t4294997321(4294997321) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/952 e 0 to 0 dl 1775764785 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1775764771.624318:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2d5:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1775764771.624331:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764771.624350:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:3.0:1775764771.624352:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:3.0:1775764771.624387:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2d5:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:3.0:1775764771.637097:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efddf80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966958592:192.168.202.146@tcp:35:dbench.0 00000002:00100000:3.0:1775764771.637130:0:19723:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f994f2e7800 x1862022966902272/t4294997338(4294997338) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/952 e 0 to 0 dl 1775764786 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1775764771.637211:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xed6:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1775764771.637224:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764771.637243:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:3.0:1775764771.637245:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000008:00010000:3.0:1775764771.637263:0:19723:0:(osc_cache.c:1167:osc_extent_make_ready()) ### extent: 00000000206bb1ce ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b03c608/0x36f9bdc9e421c12c lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f87fe expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000008:00010000:3.0:1775764771.637288:0:19723:0:(osc_cache.c:901:osc_extent_finish()) ### extent: 00000000206bb1ce ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b03c608/0x36f9bdc9e421c12c lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f87fe expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000008:00010000:3.0:1775764771.637322:0:19723:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000206bb1ce ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b03c608/0x36f9bdc9e421c12c lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f87fe expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000080:00200000:3.0:1775764771.637358:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xed6:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.637414:0:19723:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0xed6:0x0](ffff8f9952ee9148) 00000008:00010000:3.0:1775764771.637448:0:19723:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 2048 blocks 1 [cma]time 1775764770 1775764770 0 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b03c608/0x36f9bdc9e421c12c lrc: 1/0,0 mode: PW/PW res: [0x280000403:0x65f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5f87fe expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000080:00200008:3.0:1775764771.637499:0:19723:0:(file.c:447:ll_file_release()) START file COURSES.FSL:[0x200000401:0x2e4:0x0](ffff8f9952d900c8), flags 100002 00000002:00100000:3.0:1775764771.637545:0:19723:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f99463c3100 x1862022966895104/t4294997326(4294997326) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/952 e 0 to 0 dl 1775764785 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000020:00100000:3.0:1775764771.637555:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:3.0:1775764771.637560:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970680000 x1862022966959104/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:3.0:1775764771.637580:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970680000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966959104:192.168.202.146@tcp:35:dbench.0 00000100:00100000:3.0:1775764771.638033:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413d80 going to sleep for 11 seconds 00000100:00100000:3.0:1775764771.649516:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970680000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966959104:192.168.202.146@tcp:35:dbench.0 00000002:00100000:3.0:1775764771.649548:0:19723:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f99463c3100 x1862022966895104/t4294997326(4294997326) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/952 e 0 to 0 dl 1775764785 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1775764771.649639:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2e4:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1775764771.649651:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764771.649672:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:3.0:1775764771.649674:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:3.0:1775764771.649708:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2e4:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.650128:0:19723:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2e4:0x0](ffff8f9952d900c8) 00000008:00010000:3.0:1775764771.650168:0:19723:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 1 [cma]time 1775764059 1775764059 0 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000e953e3b/0x36f9bdc9e41cb0ce lrc: 1/0,0 mode: PW/PW res: [0x240000403:0x68:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef5cd expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 0 00000080:00200008:3.0:1775764771.650223:0:19723:0:(file.c:447:ll_file_release()) START file STUDENTS.FSL:[0x200000401:0x2e2:0x0](ffff8f9952dd5b88), flags 100002 00000002:00100000:3.0:1775764771.650268:0:19723:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f99463c1880 x1862022966893312/t4294997324(4294997324) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/952 e 0 to 0 dl 1775764785 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000020:00100000:3.0:1775764771.650279:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:3.0:1775764771.650284:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970680000 x1862022966959616/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:3.0:1775764771.650829:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970680000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966959616:192.168.202.146@tcp:35:dbench.0 00000100:00100000:3.0:1775764771.650918:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413d80 going to sleep for 11 seconds 00000100:00100000:3.0:1775764771.663686:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970680000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966959616:192.168.202.146@tcp:35:dbench.0 00000002:00100000:3.0:1775764771.663718:0:19723:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f99463c1880 x1862022966893312/t4294997324(4294997324) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/952 e 0 to 0 dl 1775764785 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1775764771.663813:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2e2:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1775764771.663827:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764771.663849:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:3.0:1775764771.663850:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:3.0:1775764771.663887:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2e2:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.664282:0:19723:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2e2:0x0](ffff8f9952dd5b88) 00000008:00010000:3.0:1775764771.664656:0:19723:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 1 [cma]time 1775764059 1775764059 0 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000070c338a1/0x36f9bdc9e41cb09d lrc: 1/0,0 mode: PW/PW res: [0x240000403:0x67:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef556 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 29 lru_type: 1 00000080:00200008:3.0:1775764771.664720:0:19723:0:(file.c:447:ll_file_release()) START file STUDENTS.TV:[0x200000401:0x2e0:0x0](ffff8f9952dd5348), flags 100002 00000002:00100000:3.0:1775764771.664786:0:19723:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f99463c2680 x1862022966891392/t4294997322(4294997322) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/952 e 0 to 0 dl 1775764785 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000020:00100000:3.0:1775764771.664798:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:3.0:1775764771.664803:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970680000 x1862022966960128/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:3.0:1775764771.664823:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970680000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966960128:192.168.202.146@tcp:35:dbench.0 00000100:00100000:3.0:1775764771.664909:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413d80 going to sleep for 11 seconds 00000100:00100000:0.0:1775764771.665127:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdf800 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966959872:192.168.202.146@tcp:35:dbench.0 00000002:00100000:0.0:1775764771.665145:0:19724:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f994eeaf800 x1862022966885248/t4294997315(4294997315) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/952 e 0 to 0 dl 1775764785 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1775764771.665199:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2cd:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764771.665207:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1775764771.665222:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:0.0:1775764771.665224:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:0.0:1775764771.665245:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2cd:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764771.665542:0:19724:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2cd:0x0](ffff8f9952d92a08) 00000008:00010000:0.0:1775764771.665570:0:19724:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 1 [cma]time 1775764057 1775764057 0 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000007c16d2ef/0x36f9bdc9e41cad9b lrc: 1/0,0 mode: PW/PW res: [0x240000403:0x5d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eedf4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 48 lru_type: 1 00000080:00200008:0.0:1775764771.665808:0:19724:0:(file.c:447:ll_file_release()) START file STUDENTS.YG1:[0x200000401:0x2cb:0x0](ffff8f9952e2cb08), flags 100002 00000002:00100000:0.0:1775764771.665840:0:19724:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f994f2e7b80 x1862022966884096/t4294997314(4294997314) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/952 e 0 to 0 dl 1775764785 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1775764771.665850:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 2 is allocated opc 35, max 7 00000100:00100000:0.0:1775764771.665854:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f0c9180 x1862022966960384/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1775764771.665869:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c9180 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966960384:192.168.202.146@tcp:35:dbench.0 00000100:00100000:0.0:1775764771.665907:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f3c0 going to sleep for 11 seconds 00000100:00100000:0.0:1775764771.670999:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c9180 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966960384:192.168.202.146@tcp:35:dbench.0 00000002:00100000:0.0:1775764771.671030:0:19724:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f994f2e7b80 x1862022966884096/t4294997314(4294997314) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/952 e 0 to 0 dl 1775764785 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1775764771.671106:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2cb:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764771.671119:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1775764771.671141:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:0.0:1775764771.671143:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:0.0:1775764771.671179:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2cb:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764771.671533:0:19724:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2cb:0x0](ffff8f9952e2cb08) 00000008:00010000:0.0:1775764771.671573:0:19724:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 1 [cma]time 1775764057 1775764057 0 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000050510d50/0x36f9bdc9e41cad55 lrc: 1/0,0 mode: PW/PW res: [0x240000403:0x5c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eed4c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 38 lru_type: 1 00000080:00200008:0.0:1775764771.671641:0:19724:0:(file.c:447:ll_file_release()) START file STUDENTS.XG1:[0x200000401:0x2c9:0x0](ffff8f9952e2db88), flags 100002 00000002:00100000:0.0:1775764771.671688:0:19724:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f994eeac000 x1862022966882432/t4294997312(4294997312) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/952 e 0 to 0 dl 1775764785 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1775764771.671699:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 2 is allocated opc 35, max 7 00000100:00100000:0.0:1775764771.671703:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f0c9180 x1862022966960640/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1775764771.671723:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c9180 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966960640:192.168.202.146@tcp:35:dbench.0 00000100:00100000:0.0:1775764771.671812:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f3c0 going to sleep for 11 seconds 00000100:00100000:3.0:1775764771.672034:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970680000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966960128:192.168.202.146@tcp:35:dbench.0 00000002:00100000:3.0:1775764771.672053:0:19723:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f99463c2680 x1862022966891392/t4294997322(4294997322) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/952 e 0 to 0 dl 1775764785 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1775764771.672103:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2e0:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1775764771.672111:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764771.672125:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:3.0:1775764771.672127:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:3.0:1775764771.672147:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2e0:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.672183:0:19723:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2e0:0x0](ffff8f9952dd5348) 00000008:00010000:3.0:1775764771.672205:0:19723:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 260096 blocks 515 [cma]time: 1775764059 1775764059 0 by oinfo size 260096 blocks 515 [cma]time 1775764059 1775764059 1775764659 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000e275e2c0/0x36f9bdc9e420ed6c lrc: 1/0,0 mode: PR/PR res: [0x280000403:0x66:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5cd375 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00000080:00200008:3.0:1775764771.672254:0:19723:0:(file.c:447:ll_file_release()) START file STUDENTS.MB:[0x200000401:0x2d4:0x0](ffff8f9952d95b88), flags 100002 00000002:00100000:3.0:1775764771.672280:0:19723:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f99463c1180 x1862022966889472/t4294997320(4294997320) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/952 e 0 to 0 dl 1775764785 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000020:00100000:3.0:1775764771.672303:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:3.0:1775764771.672309:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970680000 x1862022966960896/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:3.0:1775764771.672323:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970680000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966960896:192.168.202.146@tcp:35:dbench.0 00000100:00100000:3.0:1775764771.672353:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413d80 going to sleep for 11 seconds 00000100:00100000:0.0:1775764771.677465:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c9180 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966960640:192.168.202.146@tcp:35:dbench.0 00000002:00100000:0.0:1775764771.677501:0:19724:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f994eeac000 x1862022966882432/t4294997312(4294997312) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/952 e 0 to 0 dl 1775764785 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1775764771.677577:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2c9:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764771.677590:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1775764771.677622:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:0.0:1775764771.677624:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:0.0:1775764771.677658:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2c9:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764771.678120:0:19724:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2c9:0x0](ffff8f9952e2db88) 00000008:00010000:0.0:1775764771.678164:0:19724:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 1 [cma]time 1775764057 1775764057 0 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000002c55d5c6/0x36f9bdc9e41cad0f lrc: 1/0,0 mode: PW/PW res: [0x240000403:0x5b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eeca4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 38 lru_type: 1 00000080:00200008:0.0:1775764771.678227:0:19724:0:(file.c:447:ll_file_release()) START file STUDENTS.YG0:[0x200000401:0x2c7:0x0](ffff8f9952d97448), flags 100002 00000002:00100000:0.0:1775764771.678273:0:19724:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f994eead500 x1862022966880128/t4294997310(4294997310) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/952 e 0 to 0 dl 1775764785 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1775764771.678290:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 2 is allocated opc 35, max 7 00000100:00100000:0.0:1775764771.678295:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f0c9180 x1862022966961152/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1775764771.678317:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c9180 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966961152:192.168.202.146@tcp:35:dbench.0 00000100:00100000:0.0:1775764771.678389:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f3c0 going to sleep for 11 seconds 00000100:00100000:3.0:1775764771.678660:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970680000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966960896:192.168.202.146@tcp:35:dbench.0 00000002:00100000:3.0:1775764771.678679:0:19723:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f99463c1180 x1862022966889472/t4294997320(4294997320) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/952 e 0 to 0 dl 1775764785 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1775764771.678732:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2d4:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1775764771.678763:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764771.678781:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:3.0:1775764771.678783:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:3.0:1775764771.678804:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2d4:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.679116:0:19723:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2d4:0x0](ffff8f9952d95b88) 00000008:00010000:3.0:1775764771.679145:0:19723:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 1 [cma]time 1775764058 1775764058 0 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f444aa51/0x36f9bdc9e41cae97 lrc: 1/0,0 mode: PW/PW res: [0x280000403:0x60:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef055 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 139 lru_type: 1 00000080:00200008:3.0:1775764771.679193:0:19723:0:(file.c:447:ll_file_release()) START file ENROLL.VAL:[0x200000401:0x2de:0x0](ffff8f9952dd21c8), flags 100002 00000002:00100000:3.0:1775764771.679222:0:19723:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f99703dd880 x1862022966887808/t4294997318(4294997318) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/952 e 0 to 0 dl 1775764785 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000020:00100000:3.0:1775764771.679232:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:3.0:1775764771.679236:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970680000 x1862022966961408/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:3.0:1775764771.679252:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970680000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966961408:192.168.202.146@tcp:35:dbench.0 00000100:00100000:3.0:1775764771.679288:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413d80 going to sleep for 11 seconds 00000100:00100000:3.0:1775764771.684526:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970680000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966961408:192.168.202.146@tcp:35:dbench.0 00000002:00100000:3.0:1775764771.684559:0:19723:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f99703dd880 x1862022966887808/t4294997318(4294997318) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/952 e 0 to 0 dl 1775764785 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1775764771.684654:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2de:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1775764771.684667:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764771.684688:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:3.0:1775764771.684690:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:3.0:1775764771.684726:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2de:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.685191:0:19723:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2de:0x0](ffff8f9952dd21c8) 00000008:00010000:3.0:1775764771.685234:0:19723:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 1 [cma]time 1775764059 1775764059 0 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000acca1f81/0x36f9bdc9e41caffc lrc: 1/0,0 mode: PW/PW res: [0x280000403:0x65:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef3ab expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 87 lru_type: 1 00000080:00200008:3.0:1775764771.685293:0:19723:0:(file.c:447:ll_file_release()) START file ENROLL.Y03:[0x200000401:0x2dc:0x0](ffff8f9952d62a08), flags 100002 00000002:00100000:3.0:1775764771.685339:0:19723:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f99703df100 x1862022966885760/t4294997316(4294997316) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/952 e 0 to 0 dl 1775764785 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000020:00100000:3.0:1775764771.685349:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:3.0:1775764771.685354:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970680000 x1862022966961664/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:3.0:1775764771.685375:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970680000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966961664:192.168.202.146@tcp:35:dbench.0 00000100:00100000:3.0:1775764771.685447:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413d80 going to sleep for 11 seconds 00000100:00100000:0.0:1775764771.685910:0:19724:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c9180 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966961152:192.168.202.146@tcp:35:dbench.0 00000002:00100000:0.0:1775764771.685928:0:19724:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f994eead500 x1862022966880128/t4294997310(4294997310) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/952 e 0 to 0 dl 1775764785 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1775764771.685993:0:19724:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2c7:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764771.686002:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1775764771.686018:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:0.0:1775764771.686020:0:19724:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:0.0:1775764771.686045:0:19724:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2c7:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764771.686348:0:19724:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2c7:0x0](ffff8f9952d97448) 00000008:00010000:0.0:1775764771.686378:0:19724:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 1 [cma]time 1775764057 1775764057 0 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001fc9e7d4/0x36f9bdc9e41cacc9 lrc: 1/0,0 mode: PW/PW res: [0x240000403:0x5a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4eebfc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 48 lru_type: 1 00000080:00200008:0.0:1775764771.686423:0:19724:0:(file.c:447:ll_file_release()) START file STUDENTS.XG0:[0x200000401:0x2c5:0x0](ffff8f9952d8ec08), flags 100002 00000002:00100000:0.0:1775764771.686453:0:19724:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f994eeace00 x1862022966878080/t4294997308(4294997308) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/952 e 0 to 0 dl 1775764785 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1775764771.686462:0:19724:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 2 is allocated opc 35, max 7 00000100:00100000:0.0:1775764771.686467:0:19724:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f0c9180 x1862022966961920/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1775764771.686481:0:19724:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c9180 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19724:1862022966961920:192.168.202.146@tcp:35:dbench.0 00000100:00100000:0.0:1775764771.686528:0:19724:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f3c0 going to sleep for 11 seconds 00000100:00100000:3.0:1775764771.691437:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970680000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966961664:192.168.202.146@tcp:35:dbench.0 00000002:00100000:3.0:1775764771.691475:0:19723:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f99703df100 x1862022966885760/t4294997316(4294997316) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/952 e 0 to 0 dl 1775764785 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1775764771.691562:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2dc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1775764771.691574:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764771.691594:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:3.0:1775764771.691596:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:3.0:1775764771.691645:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2dc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.692067:0:19723:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2dc:0x0](ffff8f9952d62a08) 00000008:00010000:3.0:1775764771.692109:0:19723:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 1 [cma]time 1775764059 1775764059 0 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000b908cb64/0x36f9bdc9e41cafa8 lrc: 1/0,0 mode: PW/PW res: [0x280000403:0x64:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef303 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 66 lru_type: 1 00000080:00200008:3.0:1775764771.692175:0:19723:0:(file.c:447:ll_file_release()) START file ENROLL.X03:[0x200000401:0x2da:0x0](ffff8f9952d95348), flags 100002 00000002:00100000:3.0:1775764771.692224:0:19723:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f994eeae680 x1862022966883328/t4294997313(4294997313) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/952 e 0 to 0 dl 1775764785 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000020:00100000:3.0:1775764771.692235:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:3.0:1775764771.692240:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970680000 x1862022966962176/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:3.0:1775764771.692262:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970680000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966962176:192.168.202.146@tcp:35:dbench.0 00000100:00100000:3.0:1775764771.692332:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413d80 going to sleep for 11 seconds 00000100:00100000:3.0:1775764771.708526:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970680000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966962176:192.168.202.146@tcp:35:dbench.0 00000002:00100000:3.0:1775764771.708545:0:19723:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f994eeae680 x1862022966883328/t4294997313(4294997313) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/952 e 0 to 0 dl 1775764785 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1775764771.708604:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2da:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1775764771.708625:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764771.708644:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:3.0:1775764771.708645:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:3.0:1775764771.708669:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2da:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.709031:0:19723:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2da:0x0](ffff8f9952d95348) 00000008:00010000:3.0:1775764771.709061:0:19723:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 1 [cma]time 1775764058 1775764058 0 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000000b928e73/0x36f9bdc9e41caf70 lrc: 1/0,0 mode: PW/PW res: [0x280000403:0x63:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef24d expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 66 lru_type: 1 00000080:00200008:3.0:1775764771.709110:0:19723:0:(file.c:447:ll_file_release()) START file ENROLL.PX:[0x200000401:0x2d8:0x0](ffff8f9952e2ec08), flags 100002 00000002:00100000:3.0:1775764771.709141:0:19723:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f994eeafb80 x1862022966881792/t4294997311(4294997311) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/952 e 0 to 0 dl 1775764785 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000020:00100000:3.0:1775764771.709150:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:3.0:1775764771.709155:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970680000 x1862022966963200/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:3.0:1775764771.709169:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970680000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966963200:192.168.202.146@tcp:35:dbench.0 00000100:00100000:3.0:1775764771.709223:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413d80 going to sleep for 11 seconds 00000100:00100000:3.0:1775764771.715366:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970680000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966963200:192.168.202.146@tcp:35:dbench.0 00000002:00100000:3.0:1775764771.715398:0:19723:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f994eeafb80 x1862022966881792/t4294997311(4294997311) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/952 e 0 to 0 dl 1775764785 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1775764771.715450:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2d8:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1775764771.715458:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764771.715475:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:3.0:1775764771.715477:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:3.0:1775764771.715498:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2d8:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764771.716180:0:19723:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2d8:0x0](ffff8f9952e2ec08) 00000008:00010000:0.0:1775764771.716228:0:19723:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 1 [cma]time 1775764058 1775764058 0 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000022d8547f/0x36f9bdc9e41caf1c lrc: 1/0,0 mode: PW/PW res: [0x280000403:0x62:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ef190 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 76 lru_type: 1 00000080:00200008:0.0:1775764771.716288:0:19723:0:(file.c:447:ll_file_release()) START file ENROLL.DB:[0x200000401:0x2d6:0x0](ffff8f9952d96c08), flags 100002 00000002:00100000:0.0:1775764771.716351:0:19723:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f994f2e5c00 x1862022966879488/t4294997309(4294997309) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/952 e 0 to 0 dl 1775764785 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1775764771.716363:0:19723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1775764771.716368:0:19723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f0c8380 x1862022966963456/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1775764771.716399:0:19723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c8380 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966963456:192.168.202.146@tcp:35:dbench.0 00000100:00100000:0.0:1775764771.716485:0:19723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f840 going to sleep for 11 seconds 00000100:00100000:3.0:1775764771.805364:0:19723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdfb80 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19723:1862022966966528:192.168.202.146@tcp:35:dbench.0 00000002:00100000:3.0:1775764771.805606:0:19723:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f994f0aad80 x1862022966853120/t4294997261(4294997261) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/952 e 0 to 0 dl 1775764784 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1775764771.805703:0:19723:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x2bd:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1775764771.805716:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764771.805776:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:3.0:1775764771.805779:0:19723:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000008:00010000:3.0:1775764771.805798:0:19723:0:(osc_cache.c:1167:osc_extent_make_ready()) ### extent: 00000000fa5b746f ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003cb7abbf/0x36f9bdc9e41cab72 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x55:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee8de expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 83 lru_type: 1 00000008:00010000:3.0:1775764771.805827:0:19723:0:(osc_cache.c:901:osc_extent_finish()) ### extent: 00000000fa5b746f ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003cb7abbf/0x36f9bdc9e41cab72 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x55:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee8de expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 83 lru_type: 1 00000008:00010000:3.0:1775764771.805851:0:19723:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000fa5b746f ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003cb7abbf/0x36f9bdc9e41cab72 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x55:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee8de expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 83 lru_type: 1 00000080:00200000:3.0:1775764771.805890:0:19723:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x2bd:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.808263:0:19723:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2bd:0x0](ffff8f9952d14b08) 00000008:00010000:3.0:1775764771.808342:0:19723:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 260096 blocks 515 [cma]time 1775764769 1775764769 0 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003cb7abbf/0x36f9bdc9e41cab72 lrc: 1/0,0 mode: PW/PW res: [0x240000403:0x55:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e4ee8de expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 83 lru_type: 1 00000080:00200000:0.0:1775764771.808914:0:19722:0:(rw.c:1928:ll_readpage()) fast read pgno: 6 00000080:00200000:0.0:1775764771.808924:0:19722:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x266:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764771.808948:0:19722:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764771.808973:0:19722:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [6, 6] 00000008:00010000:0.0:1775764771.808987:0:19722:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764771.808991:0:19722:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0df930, osc lock ffff8f996686ee40, flags 0x0 00010000:00010000:0.0:1775764771.809010:0:19722:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 2/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 4 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 0 00010000:00010000:0.0:1775764771.809026:0:19722:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (24576 28671) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 2/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 0 00010000:00010000:0.0:1775764771.809038:0:19722:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 6/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 0 00010000:00010000:0.0:1775764771.809052:0:19722:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 6/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 0 00010000:00010000:0.0:1775764771.809059:0:19722:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 5/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 0 00000080:00200000:0.0:1775764771.809072:0:19722:0:(vvp_io.c:178:vvp_prep_size()) [0x200000401:0x266:0x0] updating i_size 2097152 00000008:00010000:0.0:1775764771.809109:0:19722:0:(osc_cache.c:2998:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8f9982040240 00000008:00010000:0.0:1775764771.809155:0:131:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8f9982040240 00010000:00010000:0.0:1775764771.809175:0:131:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (24576 28671) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 4/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 0 00000008:00100000:0.0:1775764771.809213:0:131:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8f994f0cb100 - object 0x240000403:50 offset 24576<>28672 00000100:00100000:0.0:1775764771.809268:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1775764771.809287:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966966656:192.168.202.146@tcp:3:dbench.0 00000008:00010000:1.0:1775764771.818825:0:2379:0:(osc_cache.c:901:osc_extent_finish()) ### no dlm lock: extent: ffff8f9982040240 00000020:00200000:1.0:1775764771.818841:0:2379:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1775764771.818854:0:2379:0:(osc_cache.c:496:__osc_extent_remove()) ### no dlm lock: extent: ffff8f9982040240 00000100:00100000:1.0:1775764771.818879:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966966656:192.168.202.146@tcp:3:dbench.0 00000020:00010000:0.0:1775764771.819030:0:19722:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99713e9998 (ffff8f9966dc2f38/0) at cl_lock_release():245 00000020:00010000:0.0:1775764771.819035:0:19722:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99713e9998 (ffff8f9966dc2f38/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764771.819039:0:19722:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0df930 (ffff8f9966dc2998/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764771.819048:0:19722:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 4/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 0 00010000:00010000:0.0:1775764771.819065:0:19722:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 3/0,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00000020:00010000:0.0:1775764771.819081:0:19722:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99713e9998 (ffff8f9966dc2f38/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764771.819084:0:19722:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0df930 (ffff8f9966dc2f38/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764771.819113:0:19722:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x266:0x0] ignore/verify layout 0/0, layout version 4 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764771.819300:0:19722:0:(rw.c:1928:ll_readpage()) fast read pgno: 137 00000080:00200000:0.0:1775764771.819309:0:19722:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x266:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764771.819321:0:19722:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764771.819346:0:19722:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [137, 137] 00000008:00010000:0.0:1775764771.819357:0:19722:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764771.819362:0:19722:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0df930, osc lock ffff8f996686ee40, flags 0x0 00010000:00010000:0.0:1775764771.819390:0:19722:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 2/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 4 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00010000:00010000:0.0:1775764771.819403:0:19722:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (561152 565247) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 2/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00010000:00010000:0.0:1775764771.819414:0:19722:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 6/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00010000:00010000:0.0:1775764771.819422:0:19722:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 6/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00010000:00010000:0.0:1775764771.819429:0:19722:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 5/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00000080:00200000:0.0:1775764771.819443:0:19722:0:(vvp_io.c:178:vvp_prep_size()) [0x200000401:0x266:0x0] updating i_size 2097152 00000008:00010000:0.0:1775764771.819478:0:19722:0:(osc_cache.c:2998:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8f99820403c0 00000008:00010000:0.0:1775764771.819517:0:131:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8f99820403c0 00010000:00010000:0.0:1775764771.819539:0:131:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (561152 565247) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 4/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00000008:00100000:0.0:1775764771.819581:0:131:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8f994f0ca680 - object 0x240000403:50 offset 561152<>565248 00000100:00100000:1.0:1775764771.820092:0:2380:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1775764771.820109:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966966784:192.168.202.146@tcp:3:dbench.0 00000008:00010000:1.0:1775764771.823387:0:2380:0:(osc_cache.c:901:osc_extent_finish()) ### no dlm lock: extent: ffff8f99820403c0 00000020:00200000:1.0:1775764771.823406:0:2380:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1775764771.823426:0:2380:0:(osc_cache.c:496:__osc_extent_remove()) ### no dlm lock: extent: ffff8f99820403c0 00000100:00100000:1.0:1775764771.823454:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966966784:192.168.202.146@tcp:3:dbench.0 00000020:00010000:0.0:1775764771.823680:0:19722:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99713e9998 (ffff8f9966dc2f38/0) at cl_lock_release():245 00000020:00010000:0.0:1775764771.823686:0:19722:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99713e9998 (ffff8f9966dc2f38/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764771.823690:0:19722:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0df930 (ffff8f9966dc2998/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764771.823701:0:19722:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 4/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00010000:00010000:0.0:1775764771.823717:0:19722:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 3/0,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 0 00000020:00010000:0.0:1775764771.823732:0:19722:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99713e9998 (ffff8f9966dc2f38/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764771.823735:0:19722:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0df930 (ffff8f9966dc2f38/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764771.823802:0:19722:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x266:0x0] ignore/verify layout 0/0, layout version 4 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764771.824038:0:19722:0:(rw.c:1928:ll_readpage()) fast read pgno: 80 00000080:00200000:0.0:1775764771.824047:0:19722:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x266:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764771.824061:0:19722:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764771.824088:0:19722:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [80, 80] 00000008:00010000:0.0:1775764771.824099:0:19722:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764771.824104:0:19722:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0df930, osc lock ffff8f996686ee40, flags 0x0 00010000:00010000:0.0:1775764771.824124:0:19722:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 2/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 4 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 0 00010000:00010000:0.0:1775764771.824138:0:19722:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (327680 331775) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 2/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 0 00010000:00010000:0.0:1775764771.824149:0:19722:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 6/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 0 00010000:00010000:0.0:1775764771.824158:0:19722:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 6/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 0 00010000:00010000:0.0:1775764771.824165:0:19722:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 5/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 0 00000080:00200000:0.0:1775764771.824178:0:19722:0:(vvp_io.c:178:vvp_prep_size()) [0x200000401:0x266:0x0] updating i_size 2097152 00000008:00010000:0.0:1775764771.824215:0:19722:0:(osc_cache.c:2998:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8f9982040d80 00000008:00010000:0.0:1775764771.824267:0:131:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8f9982040d80 00010000:00010000:0.0:1775764771.824287:0:131:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (327680 331775) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 4/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 0 00000008:00100000:0.0:1775764771.824324:0:131:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8f994f0cad80 - object 0x240000403:50 offset 327680<>331776 00000100:00100000:1.0:1775764771.824783:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1775764771.824803:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966966912:192.168.202.146@tcp:3:dbench.0 00000008:00010000:1.0:1775764771.829061:0:2379:0:(osc_cache.c:901:osc_extent_finish()) ### no dlm lock: extent: ffff8f9982040d80 00000020:00200000:1.0:1775764771.829078:0:2379:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1775764771.829098:0:2379:0:(osc_cache.c:496:__osc_extent_remove()) ### no dlm lock: extent: ffff8f9982040d80 00000100:00100000:1.0:1775764771.829123:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966966912:192.168.202.146@tcp:3:dbench.0 00000020:00010000:0.0:1775764771.829625:0:19722:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99713e9998 (ffff8f9966dc2f38/0) at cl_lock_release():245 00000020:00010000:0.0:1775764771.829632:0:19722:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99713e9998 (ffff8f9966dc2f38/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764771.829638:0:19722:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0df930 (ffff8f9966dc2998/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764771.829649:0:19722:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 4/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 0 00010000:00010000:0.0:1775764771.829667:0:19722:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 3/0,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 0 00000020:00010000:0.0:1775764771.829685:0:19722:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99713e9998 (ffff8f9966dc2f38/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764771.829689:0:19722:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0df930 (ffff8f9966dc2f38/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764771.829728:0:19722:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x266:0x0] ignore/verify layout 0/0, layout version 4 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764771.829861:0:19722:0:(rw.c:1928:ll_readpage()) fast read pgno: 81 00000080:00200000:0.0:1775764771.829870:0:19722:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x266:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764771.829885:0:19722:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764771.829911:0:19722:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [81, 81] 00000008:00010000:0.0:1775764771.829926:0:19722:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764771.830065:0:19722:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0df930, osc lock ffff8f996686ee40, flags 0x0 00010000:00010000:0.0:1775764771.830090:0:19722:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 2/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 4 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 0 00010000:00010000:0.0:1775764771.830103:0:19722:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (331776 335871) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 2/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 0 00010000:00010000:0.0:1775764771.830115:0:19722:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 6/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 0 00010000:00010000:0.0:1775764771.830123:0:19722:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 6/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 0 00010000:00010000:0.0:1775764771.830130:0:19722:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 5/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 0 00000080:00200000:0.0:1775764771.830144:0:19722:0:(vvp_io.c:178:vvp_prep_size()) [0x200000401:0x266:0x0] updating i_size 2097152 00000008:00010000:0.0:1775764771.830185:0:19722:0:(osc_cache.c:2998:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8f9982040780 00000008:00010000:0.0:1775764771.830233:0:131:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8f9982040780 00010000:00010000:0.0:1775764771.830255:0:131:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (331776 335871) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 4/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 0 00000008:00100000:0.0:1775764771.831111:0:131:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8f994f0cad80 - object 0x240000403:50 offset 331776<>335872 00000100:00100000:1.0:1775764771.831298:0:2380:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1775764771.831320:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966967040:192.168.202.146@tcp:3:dbench.0 00000008:00010000:1.0:1775764771.835918:0:2380:0:(osc_cache.c:901:osc_extent_finish()) ### no dlm lock: extent: ffff8f9982040780 00000020:00200000:1.0:1775764771.836011:0:2380:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1775764771.836075:0:2380:0:(osc_cache.c:496:__osc_extent_remove()) ### no dlm lock: extent: ffff8f9982040780 00000100:00100000:1.0:1775764771.836102:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966967040:192.168.202.146@tcp:3:dbench.0 00000020:00010000:0.0:1775764771.836619:0:19722:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99713e9998 (ffff8f9966dc2f38/0) at cl_lock_release():245 00000020:00010000:0.0:1775764771.836626:0:19722:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99713e9998 (ffff8f9966dc2f38/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764771.836630:0:19722:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0df930 (ffff8f9966dc2998/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764771.836639:0:19722:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 4/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 0 00010000:00010000:0.0:1775764771.836653:0:19722:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 3/0,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 0 00000020:00010000:0.0:1775764771.836669:0:19722:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99713e9998 (ffff8f9966dc2f38/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764771.836672:0:19722:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0df930 (ffff8f9966dc2f38/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764771.836704:0:19722:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x266:0x0] ignore/verify layout 0/0, layout version 4 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764771.836868:0:19722:0:(rw.c:1928:ll_readpage()) fast read pgno: 320 00000080:00200000:0.0:1775764771.836877:0:19722:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x266:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764771.836890:0:19722:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764771.836911:0:19722:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [320, 320] 00000008:00010000:0.0:1775764771.836920:0:19722:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764771.836924:0:19722:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f0df930, osc lock ffff8f996686ee40, flags 0x0 00010000:00010000:0.0:1775764771.837088:0:19722:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4f14cf5/0x36f9bdc9e4211ec0 lrc: 2/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 1232896->1236991) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 0 00010000:00010000:0.0:1775764771.837101:0:19722:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (1310720 1314815) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4f14cf5/0x36f9bdc9e4211ec0 lrc: 2/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1232896->1236991) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 0 00010000:00010000:0.0:1775764771.837110:0:19722:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4f14cf5/0x36f9bdc9e4211ec0 lrc: 6/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1232896->1236991) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 0 00010000:00010000:0.0:1775764771.837116:0:19722:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4f14cf5/0x36f9bdc9e4211ec0 lrc: 6/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1232896->1236991) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 0 00010000:00010000:0.0:1775764771.837125:0:19722:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4f14cf5/0x36f9bdc9e4211ec0 lrc: 5/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1232896->1236991) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 0 00000080:00200000:0.0:1775764771.837136:0:19722:0:(vvp_io.c:178:vvp_prep_size()) [0x200000401:0x266:0x0] updating i_size 2097152 00000008:00010000:0.0:1775764771.837171:0:19722:0:(osc_cache.c:2998:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8f9982040480 00000008:00010000:0.0:1775764771.837212:0:131:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8f9982040480 00010000:00010000:0.0:1775764771.837229:0:131:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (1310720 1314815) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4f14cf5/0x36f9bdc9e4211ec0 lrc: 4/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1232896->1236991) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 0 00000008:00100000:0.0:1775764771.837260:0:131:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8f994f0ca680 - object 0x240000403:50 offset 1310720<>1314816 00000100:00100000:1.0:1775764771.837400:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1775764771.837416:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966967168:192.168.202.146@tcp:3:dbench.0 00000008:00010000:1.0:1775764771.840849:0:2379:0:(osc_cache.c:901:osc_extent_finish()) ### no dlm lock: extent: ffff8f9982040480 00000020:00200000:1.0:1775764771.840862:0:2379:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1775764771.840877:0:2379:0:(osc_cache.c:496:__osc_extent_remove()) ### no dlm lock: extent: ffff8f9982040480 00000100:00100000:1.0:1775764771.840895:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966967168:192.168.202.146@tcp:3:dbench.0 00000020:00010000:0.0:1775764771.841271:0:19722:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99713e9998 (ffff8f9966dc2f38/0) at cl_lock_release():245 00000020:00010000:0.0:1775764771.841274:0:19722:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99713e9998 (ffff8f9966dc2f38/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764771.841278:0:19722:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f0df930 (ffff8f9966dc2998/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764771.841285:0:19722:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4f14cf5/0x36f9bdc9e4211ec0 lrc: 4/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1232896->1236991) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 0 00010000:00010000:0.0:1775764771.841662:0:19722:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4f14cf5/0x36f9bdc9e4211ec0 lrc: 3/0,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1232896->1236991) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 0 00000020:00010000:0.0:1775764771.841679:0:19722:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99713e9998 (ffff8f9966dc2f38/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764771.841683:0:19722:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f0df930 (ffff8f9966dc2f38/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764771.841724:0:19722:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x266:0x0] ignore/verify layout 0/0, layout version 4 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764771.841900:0:19722:0:(rw.c:1928:ll_readpage()) fast read pgno: 8 00000080:00200000:1.0:1775764771.841927:0:19722:0:(rw.c:1928:ll_readpage()) fast read pgno: 30 00000080:00200000:1.0:1775764771.842029:0:19722:0:(rw.c:1928:ll_readpage()) fast read pgno: 25 00000080:00200000:1.0:1775764771.842057:0:19722:0:(rw.c:1928:ll_readpage()) fast read pgno: 21 00000080:00200000:1.0:1775764771.842086:0:19722:0:(rw.c:1928:ll_readpage()) fast read pgno: 2 00000080:00200000:1.0:1775764771.842133:0:19722:0:(rw.c:1928:ll_readpage()) fast read pgno: 150 00000080:00200000:1.0:1775764771.842140:0:19722:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x266:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764771.842155:0:19722:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764771.842178:0:19722:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [150, 150] 00000008:00010000:1.0:1775764771.842192:0:19722:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764771.842196:0:19722:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f1c7430, osc lock ffff8f995875db50, flags 0x0 00010000:00010000:1.0:1775764771.842216:0:19722:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 2/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 4 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 0 00010000:00010000:1.0:1775764771.842229:0:19722:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (614400 618495) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 2/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 0 00010000:00010000:1.0:1775764771.842242:0:19722:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 6/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 0 00010000:00010000:1.0:1775764771.842249:0:19722:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 6/2,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 0 00010000:00010000:1.0:1775764771.842256:0:19722:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 5/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 0 00000080:00200000:1.0:1775764771.842269:0:19722:0:(vvp_io.c:178:vvp_prep_size()) [0x200000401:0x266:0x0] updating i_size 2097152 00000008:00010000:1.0:1775764771.842624:0:19722:0:(osc_cache.c:2998:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8f9970a08a80 00000008:00010000:1.0:1775764771.842662:0:21:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8f9970a08a80 00010000:00010000:1.0:1775764771.842678:0:21:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (614400 618495) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 4/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 0 00000008:00100000:1.0:1775764771.842707:0:21:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8f9970b53100 - object 0x240000403:50 offset 614400<>618496 00000100:00100000:0.0:1775764771.842766:0:2380:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1775764771.842785:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970b53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966967296:192.168.202.146@tcp:3:dbench.0 00000008:00010000:1.0:1775764771.845993:0:2380:0:(osc_cache.c:901:osc_extent_finish()) ### no dlm lock: extent: ffff8f9970a08a80 00000020:00200000:1.0:1775764771.846009:0:2380:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1775764771.846028:0:2380:0:(osc_cache.c:496:__osc_extent_remove()) ### no dlm lock: extent: ffff8f9970a08a80 00000100:00100000:1.0:1775764771.846054:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970b53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966967296:192.168.202.146@tcp:3:dbench.0 00000020:00010000:3.0:1775764771.846188:0:19722:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705ba5d8 (ffff8f99705b8638/0) at cl_lock_release():245 00000020:00010000:3.0:1775764771.846193:0:19722:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705ba5d8 (ffff8f99705b8638/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764771.846196:0:19722:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f1c7430 (ffff8f99705b8998/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764771.846204:0:19722:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 4/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 0 00010000:00010000:3.0:1775764771.846218:0:19722:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 3/0,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 0 00000020:00010000:3.0:1775764771.846233:0:19722:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705ba5d8 (ffff8f99705b8638/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764771.846236:0:19722:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f1c7430 (ffff8f99705b8638/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764771.846259:0:19722:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x266:0x0] ignore/verify layout 0/0, layout version 4 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764771.846354:0:19722:0:(rw.c:1928:ll_readpage()) fast read pgno: 9 00000080:00200008:3.0:1775764771.846799:0:19722:0:(file.c:447:ll_file_release()) START file client.txt:[0x200000401:0x262:0x0](ffff8f9952db1148), flags 100000 00000002:00100000:3.0:1775764771.846869:0:19722:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f994f0c9880 x1862022950240000/t4294969439(4294969439) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 912/1000 e 0 to 0 dl 1775764065 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000020:00100000:3.0:1775764771.846880:0:19722:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:3.0:1775764771.846885:0:19722:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970680000 x1862022966967552/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:3.0:1775764771.846906:0:19722:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970680000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19722:1862022966967552:192.168.202.146@tcp:35:dbench.0 00000100:00100000:3.0:1775764771.846981:0:19722:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413d80 going to sleep for 11 seconds 00000100:00100000:3.0:1775764771.850978:0:19722:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970680000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19722:1862022966967552:192.168.202.146@tcp:35:dbench.0 00000002:00100000:3.0:1775764771.851014:0:19722:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f994f0c9880 x1862022950240000/t4294969439(4294969439) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 912/1000 e 0 to 0 dl 1775764065 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000080:00200008:3.0:1775764771.851151:0:19722:0:(file.c:447:ll_file_release()) START file dbench:[0x200000401:0x263:0x0](ffff8f9952e62a08), flags 100040 00000002:00100000:3.0:1775764771.851218:0:19722:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f9970edf480 x1862022950222464/t4294969418(4294969418) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/952 e 0 to 0 dl 1775764063 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'ld-linux-x86-64.0' uid:0 gid:0 projid:0 00000020:00100000:3.0:1775764771.851229:0:19722:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:3.0:1775764771.851235:0:19722:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970680000 x1862022966967808/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:3.0:1775764771.851257:0:19722:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970680000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19722:1862022966967808:192.168.202.146@tcp:35:dbench.0 00000100:00100000:3.0:1775764771.851331:0:19722:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413d80 going to sleep for 11 seconds 00000100:00100000:3.0:1775764771.854296:0:19722:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970680000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19722:1862022966967808:192.168.202.146@tcp:35:dbench.0 00000002:00100000:3.0:1775764771.854330:0:19722:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f9970edf480 x1862022950222464/t4294969418(4294969418) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/952 e 0 to 0 dl 1775764063 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'ld-linux-x86-64.0' uid:0 gid:0 projid:0 00000080:00200008:3.0:1775764771.854428:0:19722:0:(file.c:447:ll_file_release()) START file ld-linux-x86-64.so.2:[0x200000401:0x267:0x0](ffff8f9952e63248), flags 100040 00000002:00100000:3.0:1775764771.854475:0:19722:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f994f0a9c00 x1862022950236288/t4294969436(4294969436) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 880/952 e 0 to 0 dl 1775764064 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'chroot.0' uid:0 gid:0 projid:0 00000020:00100000:3.0:1775764771.854484:0:19722:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:3.0:1775764771.854489:0:19722:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970680000 x1862022966968064/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:3.0:1775764771.854508:0:19722:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970680000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19722:1862022966968064:192.168.202.146@tcp:35:dbench.0 00000100:00100000:3.0:1775764771.854640:0:19722:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413d80 going to sleep for 11 seconds 00000100:00100000:3.0:1775764771.863187:0:19722:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970680000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19722:1862022966968064:192.168.202.146@tcp:35:dbench.0 00000002:00100000:3.0:1775764771.863222:0:19722:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f994f0a9c00 x1862022950236288/t4294969436(4294969436) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 880/952 e 0 to 0 dl 1775764064 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'chroot.0' uid:0 gid:0 projid:0 00000080:00200008:3.0:1775764771.863298:0:19722:0:(file.c:447:ll_file_release()) START file libpopt.so.0:[0x200000401:0x265:0x0](ffff8f9952da1148), flags 100000 00000002:00100000:3.0:1775764771.863380:0:19722:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f994f0c9f80 x1862022950239104/t4294969437(4294969437) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 872/952 e 0 to 0 dl 1775764065 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000020:00100000:3.0:1775764771.863392:0:19722:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:3.0:1775764771.863397:0:19722:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970680000 x1862022966968320/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:3.0:1775764771.863424:0:19722:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970680000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19722:1862022966968320:192.168.202.146@tcp:35:dbench.0 00000100:00100000:3.0:1775764771.863503:0:19722:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413d80 going to sleep for 11 seconds 00000100:00100000:3.0:1775764771.867130:0:19722:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970680000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19722:1862022966968320:192.168.202.146@tcp:35:dbench.0 00000002:00100000:3.0:1775764771.867160:0:19722:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f994f0c9f80 x1862022950239104/t4294969437(4294969437) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 872/952 e 0 to 0 dl 1775764065 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000080:00200008:3.0:1775764771.867248:0:19722:0:(file.c:447:ll_file_release()) START file libc.so.6:[0x200000401:0x266:0x0](ffff8f9952e642c8), flags 100000 00000002:00100000:3.0:1775764771.867309:0:19722:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f994f0c9c00 x1862022950239360/t4294969438(4294969438) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 896/976 e 0 to 0 dl 1775764065 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000020:00100000:3.0:1775764771.867320:0:19722:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:3.0:1775764771.867325:0:19722:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970680000 x1862022966968576/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:3.0:1775764771.867345:0:19722:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970680000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19722:1862022966968576:192.168.202.146@tcp:35:dbench.0 00000100:00100000:3.0:1775764771.867424:0:19722:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413d80 going to sleep for 11 seconds 00000100:00100000:3.0:1775764771.883951:0:19722:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970680000 pname:cluuid:pid:xid:nid:opc:job dbench:3f83546c-01a9-4156-8a97-44c446245d3e:19722:1862022966968576:192.168.202.146@tcp:35:dbench.0 00000002:00100000:3.0:1775764771.883984:0:19722:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f994f0c9c00 x1862022950239360/t4294969438(4294969438) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 896/976 e 0 to 0 dl 1775764065 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1775764771.946279:0:19698:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200008:0.0:1775764771.946296:0:19698:0:(file.c:6226:ll_getattr_dentry()) START file d8.sanity-pfl:[0x200000401:0x261:0x0](ffff8f9952e61988), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764771.946303:0:19698:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988),name=d8.sanity-pfl 00000002:00010000:0.0:1775764771.946323:0:19698:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x261:0x0]) in obj [0x200000401:0x261:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764771.946342:0:19698:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:0.0:1775764771.946356:0:19698:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:0.0:1775764771.946364:0:19698:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x261:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764771.946371:0:19698:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x261:0x0](ffff8f9952e61988) 00000080:00010000:0.0:1775764771.946385:0:19698:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x261:0x0](ffff8f9952e61988) for lock 0x36f9bdc9e421c5d9 00000080:00010000:0.0:1775764771.946392:0:19698:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c5d9 from it ffffafbac19c7cb8 00010000:00010000:0.0:1775764771.946395:0:19698:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:0.0:1775764771.946403:0:19698:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:0.0:1775764771.946427:0:19698:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200008:0.0:1775764771.946432:0:19698:0:(file.c:6226:ll_getattr_dentry()) START file d8.sanity-pfl:[0x200000401:0x261:0x0](ffff8f9952e61988), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764771.946436:0:19698:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988),name=d8.sanity-pfl 00000002:00010000:0.0:1775764771.946441:0:19698:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x261:0x0]) in obj [0x200000401:0x261:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764771.946447:0:19698:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:0.0:1775764771.946458:0:19698:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:0.0:1775764771.946463:0:19698:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x261:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764771.946466:0:19698:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x261:0x0](ffff8f9952e61988) 00000080:00010000:0.0:1775764771.946469:0:19698:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x261:0x0](ffff8f9952e61988) for lock 0x36f9bdc9e421c5d9 00000080:00010000:0.0:1775764771.946472:0:19698:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c5d9 from it ffffafbac19c7cb8 00010000:00010000:0.0:1775764771.946474:0:19698:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:0.0:1775764771.946480:0:19698:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:0.0:1775764771.946493:0:19698:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.946497:0:19698:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.946510:0:19698:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=ps, dir=[0x200000401:0x261:0x0](ffff8f9952e61988), flags=1 00000080:00200000:0.0:1775764771.946514:0:19698:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=ps, dir=[0x200000401:0x261:0x0](ffff8f9952e61988), intent=getattr 00000002:00010000:0.0:1775764771.946525:0:19698:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ps,[0x0:0x0:0x0]) in obj [0x200000401:0x261:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764771.946569:0:19698:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c603 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19698 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.946576:0:19698:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c603 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19698 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.946585:0:19698:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c603 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19698 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764771.946623:0:19698:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0cb100 pname:cluuid:pid:xid:nid:opc:job bash:3f83546c-01a9-4156-8a97-44c446245d3e:19698:1862022966969600:192.168.202.146@tcp:101:bash.0 00000100:00100000:0.0:1775764771.946790:0:19698:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f180 going to sleep for 11 seconds 00000100:00100000:1.0:1775764771.950927:0:19698:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0cb100 pname:cluuid:pid:xid:nid:opc:job bash:3f83546c-01a9-4156-8a97-44c446245d3e:19698:1862022966969600:192.168.202.146@tcp:101:bash.0 00010000:00010000:1.0:1775764771.951020:0:19698:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c603 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19698 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.951033:0:19698:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c603 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 19698 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.951040:0:19698:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c603 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 19698 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.951047:0:19698:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c603 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 19698 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.951055:0:19698:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c603 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 19698 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.951062:0:19698:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99821cc800 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:1.0:1775764771.951067:0:19698:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c603 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 19698 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.951074:0:19698:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c603 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 19698 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.951081:0:19698:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c603 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 19698 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.951087:0:19698:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1775764771.951093:0:19698:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c603 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 19698 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.951099:0:19698:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c603 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 19698 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764771.951115:0:19698:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=7, status=0 req@ffff8f994f0cb100 x1862022966969600/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/584 e 0 to 0 dl 1775764787 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'bash.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764771.951128:0:19698:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x261:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1775764771.951153:0:19698:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:1.0:1775764771.951165:0:19698:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00010000:1.0:1775764771.951173:0:19698:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c5d9 from it ffffafbac19c7930 00010000:00010000:1.0:1775764771.951176:0:19698:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:1.0:1775764771.951184:0:19698:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:0.0:1775764771.954454:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.954474:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=glibc-hwcaps, flags=17 00000080:00200000:0.0:1775764771.954486:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.954491:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:0.0:1775764771.954499:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.954502:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:0.0:1775764771.954510:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.954513:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:0.0:1775764771.954521:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.954524:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:0.0:1775764771.954533:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.954537:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:0.0:1775764771.954547:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.954550:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:0.0:1775764771.954560:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.954562:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:0.0:1775764771.954569:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764771.954580:0:19895:0:(namei.c:1399:ll_atomic_open()) VFS Op:name=libprocps.so.7, dir=[0x200000401:0x261:0x0](ffff8f9952e61988), file ffff8f9959680c00, open_flags 8000, mode 0 opened 0 00000080:00200000:0.0:1775764771.954587:0:19895:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=libprocps.so.7, dir=[0x200000401:0x261:0x0](ffff8f9952e61988), intent=open 00000002:00010000:0.0:1775764771.954622:0:19895:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: libprocps.so.7,[0x0:0x0:0x0]) in obj [0x200000401:0x261:0x0], intent: open flags 0100001 00010000:00010000:0.0:1775764771.954692:0:19895:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cde00/0x36f9bdc9e421c60a lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764771.954701:0:19895:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cde00/0x36f9bdc9e421c60a lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1775764771.954715:0:19895:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1775764771.954728:0:19895:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f0c8380 x1862022966969856/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4624/135824 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1775764771.954761:0:19895:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cde00/0x36f9bdc9e421c60a lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764771.954785:0:19895:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c8380 pname:cluuid:pid:xid:nid:opc:job ps:3f83546c-01a9-4156-8a97-44c446245d3e:19895:1862022966969856:192.168.202.146@tcp:101:ps.0 00000100:00100000:0.0:1775764771.954963:0:19895:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16ff00 going to sleep for 11 seconds 00000100:00100000:1.0:1775764771.958048:0:19895:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c8380 pname:cluuid:pid:xid:nid:opc:job ps:3f83546c-01a9-4156-8a97-44c446245d3e:19895:1862022966969856:192.168.202.146@tcp:101:ps.0 00010000:00010000:1.0:1775764771.958073:0:19895:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cde00/0x36f9bdc9e421c60a lrc: 4/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.958088:0:19895:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cde00/0x36f9bdc9e421c60a lrc: 4/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.958095:0:19895:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cde00/0x36f9bdc9e421c60a lrc: 4/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.958102:0:19895:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cde00/0x36f9bdc9e421c60a lrc: 3/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.958112:0:19895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cde00/0x36f9bdc9e421c60a lrc: 4/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.958119:0:19895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99821cde00 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:1.0:1775764771.958123:0:19895:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cde00/0x36f9bdc9e421c60a lrc: 4/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.958130:0:19895:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cde00/0x36f9bdc9e421c60a lrc: 4/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.958137:0:19895:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cde00/0x36f9bdc9e421c60a lrc: 5/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.958143:0:19895:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1775764771.958150:0:19895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cde00/0x36f9bdc9e421c60a lrc: 3/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.958157:0:19895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cde00/0x36f9bdc9e421c60a lrc: 0/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764771.958175:0:19895:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=7, status=0 req@ffff8f994f0c8380 x1862022966969856/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4624/608 e 0 to 0 dl 1775764787 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ps.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764771.958190:0:19895:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0x261:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1775764771.958218:0:19895:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764771.958231:0:19895:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00010000:1.0:1775764771.958246:0:19895:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c5d9 from it ffffafbac1f37998 00010000:00010000:1.0:1775764771.958250:0:19895:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764771.958259:0:19895:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:1.0:1775764771.958731:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.958764:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=glibc-hwcaps, flags=17 00000080:00200000:1.0:1775764771.958778:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.958782:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1775764771.958791:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.958795:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1775764771.958800:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.958802:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1775764771.958807:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.958809:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1775764771.958814:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.958816:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:1.0:1775764771.958820:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.958822:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:1.0:1775764771.958827:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.958828:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:1.0:1775764771.958833:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.958842:0:19895:0:(namei.c:1399:ll_atomic_open()) VFS Op:name=libsystemd.so.0, dir=[0x200000401:0x261:0x0](ffff8f9952e61988), file ffff8f9945fdf680, open_flags 8000, mode 0 opened 0 00000080:00200000:1.0:1775764771.958847:0:19895:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=libsystemd.so.0, dir=[0x200000401:0x261:0x0](ffff8f9952e61988), intent=open 00000002:00010000:1.0:1775764771.958874:0:19895:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: libsystemd.so.0,[0x0:0x0:0x0]) in obj [0x200000401:0x261:0x0], intent: open flags 0100001 00010000:00010000:1.0:1775764771.958920:0:19895:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c611 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.958925:0:19895:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c611 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764771.958933:0:19895:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764771.958938:0:19895:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970b53100 x1862022966970112/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4624/135824 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764771.958944:0:19895:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c611 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764771.958962:0:19895:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970b53100 pname:cluuid:pid:xid:nid:opc:job ps:3f83546c-01a9-4156-8a97-44c446245d3e:19895:1862022966970112:192.168.202.146@tcp:101:ps.0 00000100:00100000:1.0:1775764771.959123:0:19895:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f997130d6c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764771.963452:0:19895:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970b53100 pname:cluuid:pid:xid:nid:opc:job ps:3f83546c-01a9-4156-8a97-44c446245d3e:19895:1862022966970112:192.168.202.146@tcp:101:ps.0 00010000:00010000:1.0:1775764771.963475:0:19895:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c611 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.963487:0:19895:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c611 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.963495:0:19895:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c611 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.963502:0:19895:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c611 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.963510:0:19895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c611 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.963520:0:19895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970640000 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:1.0:1775764771.963525:0:19895:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c611 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.963532:0:19895:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c611 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.963538:0:19895:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c611 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.963543:0:19895:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1775764771.963548:0:19895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c611 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.963552:0:19895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c611 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764771.963566:0:19895:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=7, status=0 req@ffff8f9970b53100 x1862022966970112/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4624/608 e 0 to 0 dl 1775764787 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ps.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764771.963576:0:19895:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0x261:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1775764771.963666:0:19895:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:1.0:1775764771.963676:0:19895:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00010000:1.0:1775764771.963682:0:19895:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c5d9 from it ffffafbac1f37998 00010000:00010000:1.0:1775764771.963684:0:19895:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:1.0:1775764771.963693:0:19895:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:1.0:1775764771.964252:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.964260:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=glibc-hwcaps, flags=17 00000080:00200000:1.0:1775764771.964267:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.964269:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1775764771.964274:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.964276:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1775764771.964282:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.964284:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1775764771.964289:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.964300:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1775764771.964306:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.964308:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:1.0:1775764771.964313:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.964315:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:1.0:1775764771.964319:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.964320:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:1.0:1775764771.964325:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.964332:0:19895:0:(namei.c:1399:ll_atomic_open()) VFS Op:name=libdl.so.2, dir=[0x200000401:0x261:0x0](ffff8f9952e61988), file ffff8f99613abc80, open_flags 8000, mode 0 opened 0 00000080:00200000:1.0:1775764771.964336:0:19895:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=libdl.so.2, dir=[0x200000401:0x261:0x0](ffff8f9952e61988), intent=open 00000002:00010000:1.0:1775764771.964352:0:19895:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: libdl.so.2,[0x0:0x0:0x0]) in obj [0x200000401:0x261:0x0], intent: open flags 0100001 00010000:00010000:1.0:1775764771.964394:0:19895:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c618 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.964400:0:19895:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c618 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764771.964407:0:19895:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764771.964412:0:19895:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970b53100 x1862022966970368/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4624/135824 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764771.964418:0:19895:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c618 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764771.964433:0:19895:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970b53100 pname:cluuid:pid:xid:nid:opc:job ps:3f83546c-01a9-4156-8a97-44c446245d3e:19895:1862022966970368:192.168.202.146@tcp:101:ps.0 00000100:00100000:1.0:1775764771.964572:0:19895:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f997130d6c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764771.970889:0:19895:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970b53100 pname:cluuid:pid:xid:nid:opc:job ps:3f83546c-01a9-4156-8a97-44c446245d3e:19895:1862022966970368:192.168.202.146@tcp:101:ps.0 00010000:00010000:1.0:1775764771.970913:0:19895:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c618 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.970926:0:19895:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c618 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.970942:0:19895:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c618 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.970949:0:19895:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c618 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.970956:0:19895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c618 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.970962:0:19895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970640200 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:1.0:1775764771.970966:0:19895:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c618 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.970977:0:19895:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c618 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.970983:0:19895:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c618 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.970990:0:19895:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1775764771.970997:0:19895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c618 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.971004:0:19895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c618 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764771.971023:0:19895:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=7, status=0 req@ffff8f9970b53100 x1862022966970368/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4624/608 e 0 to 0 dl 1775764787 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ps.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764771.971037:0:19895:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0x261:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1775764771.971066:0:19895:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:1.0:1775764771.971078:0:19895:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00010000:1.0:1775764771.971087:0:19895:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c5d9 from it ffffafbac1f37998 00010000:00010000:1.0:1775764771.971090:0:19895:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:1.0:1775764771.971097:0:19895:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:1.0:1775764771.971436:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.971445:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=glibc-hwcaps, flags=17 00000080:00200000:1.0:1775764771.971454:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.971458:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1775764771.971466:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.971468:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1775764771.971476:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.971479:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1775764771.971486:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.971490:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1775764771.971500:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.971504:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:1.0:1775764771.971511:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.971514:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:1.0:1775764771.971522:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.971525:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:1.0:1775764771.971531:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.971535:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=libc.so.6, flags=257 00000080:00200000:1.0:1775764771.971710:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.971715:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=glibc-hwcaps, flags=17 00000080:00200000:1.0:1775764771.971723:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.971726:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1775764771.971734:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.971759:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1775764771.971768:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.971771:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1775764771.971777:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.971780:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1775764771.971787:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.971790:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:1.0:1775764771.971797:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.971800:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:1.0:1775764771.971807:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.971809:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:1.0:1775764771.971817:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.971828:0:19895:0:(namei.c:1399:ll_atomic_open()) VFS Op:name=librt.so.1, dir=[0x200000401:0x261:0x0](ffff8f9952e61988), file ffff8f99439cac00, open_flags 8000, mode 0 opened 0 00000080:00200000:1.0:1775764771.971834:0:19895:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=librt.so.1, dir=[0x200000401:0x261:0x0](ffff8f9952e61988), intent=open 00000002:00010000:1.0:1775764771.971867:0:19895:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: librt.so.1,[0x0:0x0:0x0]) in obj [0x200000401:0x261:0x0], intent: open flags 0100001 00010000:00010000:1.0:1775764771.971919:0:19895:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421c61f lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.971927:0:19895:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421c61f lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764771.971938:0:19895:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764771.971945:0:19895:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970b53100 x1862022966970624/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4624/135824 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764771.971953:0:19895:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421c61f lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764771.971975:0:19895:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970b53100 pname:cluuid:pid:xid:nid:opc:job ps:3f83546c-01a9-4156-8a97-44c446245d3e:19895:1862022966970624:192.168.202.146@tcp:101:ps.0 00000100:00100000:1.0:1775764771.972146:0:19895:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f997130d6c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764771.979958:0:19895:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970b53100 pname:cluuid:pid:xid:nid:opc:job ps:3f83546c-01a9-4156-8a97-44c446245d3e:19895:1862022966970624:192.168.202.146@tcp:101:ps.0 00010000:00010000:1.0:1775764771.979976:0:19895:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421c61f lrc: 4/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.979988:0:19895:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421c61f lrc: 4/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.979996:0:19895:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421c61f lrc: 4/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.980003:0:19895:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421c61f lrc: 3/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.980011:0:19895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421c61f lrc: 4/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.980017:0:19895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970641a00 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:1.0:1775764771.980022:0:19895:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421c61f lrc: 4/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.980029:0:19895:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421c61f lrc: 4/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.980037:0:19895:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421c61f lrc: 5/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.980044:0:19895:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1775764771.980050:0:19895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421c61f lrc: 3/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.980056:0:19895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421c61f lrc: 0/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764771.980076:0:19895:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=7, status=0 req@ffff8f9970b53100 x1862022966970624/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4624/608 e 0 to 0 dl 1775764787 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ps.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764771.980092:0:19895:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0x261:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1775764771.980121:0:19895:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:1.0:1775764771.980133:0:19895:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00010000:1.0:1775764771.980143:0:19895:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c5d9 from it ffffafbac1f37998 00010000:00010000:1.0:1775764771.980146:0:19895:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:1.0:1775764771.980155:0:19895:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:1.0:1775764771.980467:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.980476:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=glibc-hwcaps, flags=17 00000080:00200000:1.0:1775764771.980482:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.980484:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1775764771.980489:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.980491:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1775764771.980495:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.980497:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1775764771.980501:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.980503:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1775764771.980507:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.980509:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:1.0:1775764771.980513:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.980515:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:1.0:1775764771.980519:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.980520:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:1.0:1775764771.980524:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.980532:0:19895:0:(namei.c:1399:ll_atomic_open()) VFS Op:name=liblzma.so.5, dir=[0x200000401:0x261:0x0](ffff8f9952e61988), file ffff8f99439ca000, open_flags 8000, mode 0 opened 0 00000080:00200000:1.0:1775764771.980536:0:19895:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=liblzma.so.5, dir=[0x200000401:0x261:0x0](ffff8f9952e61988), intent=open 00000002:00010000:1.0:1775764771.980549:0:19895:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: liblzma.so.5,[0x0:0x0:0x0]) in obj [0x200000401:0x261:0x0], intent: open flags 0100001 00010000:00010000:1.0:1775764771.980585:0:19895:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c626 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.980719:0:19895:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c626 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764771.980732:0:19895:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764771.980791:0:19895:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970b53100 x1862022966970880/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4624/135824 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764771.980802:0:19895:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c626 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764771.980824:0:19895:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970b53100 pname:cluuid:pid:xid:nid:opc:job ps:3f83546c-01a9-4156-8a97-44c446245d3e:19895:1862022966970880:192.168.202.146@tcp:101:ps.0 00000100:00100000:1.0:1775764771.980983:0:19895:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f997130d6c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764771.984435:0:19895:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970b53100 pname:cluuid:pid:xid:nid:opc:job ps:3f83546c-01a9-4156-8a97-44c446245d3e:19895:1862022966970880:192.168.202.146@tcp:101:ps.0 00010000:00010000:1.0:1775764771.984457:0:19895:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c626 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.984468:0:19895:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c626 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.984475:0:19895:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c626 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.984481:0:19895:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c626 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.984494:0:19895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c626 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.984500:0:19895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970640200 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:1.0:1775764771.984505:0:19895:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c626 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.984511:0:19895:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c626 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.984519:0:19895:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c626 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.984526:0:19895:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1775764771.984532:0:19895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c626 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.984538:0:19895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c626 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764771.984558:0:19895:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=7, status=0 req@ffff8f9970b53100 x1862022966970880/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4624/608 e 0 to 0 dl 1775764787 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ps.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764771.984572:0:19895:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0x261:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1775764771.984597:0:19895:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:1.0:1775764771.984626:0:19895:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00010000:1.0:1775764771.984642:0:19895:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c5d9 from it ffffafbac1f37998 00010000:00010000:1.0:1775764771.984646:0:19895:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:1.0:1775764771.984654:0:19895:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00200000:1.0:1775764771.985020:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.985029:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=glibc-hwcaps, flags=17 00000080:00200000:1.0:1775764771.985036:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.985039:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1775764771.985044:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.985046:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1775764771.985050:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.985051:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1775764771.985056:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.985057:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1775764771.985061:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.985063:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:1.0:1775764771.985067:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.985069:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:1.0:1775764771.985073:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.985075:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:1.0:1775764771.985079:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.985087:0:19895:0:(namei.c:1399:ll_atomic_open()) VFS Op:name=liblz4.so.1, dir=[0x200000401:0x261:0x0](ffff8f9952e61988), file ffff8f99613abe00, open_flags 8000, mode 0 opened 0 00000080:00200000:1.0:1775764771.985091:0:19895:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=liblz4.so.1, dir=[0x200000401:0x261:0x0](ffff8f9952e61988), intent=open 00000002:00010000:1.0:1775764771.985106:0:19895:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: liblz4.so.1,[0x0:0x0:0x0]) in obj [0x200000401:0x261:0x0], intent: open flags 0100001 00010000:00010000:1.0:1775764771.985145:0:19895:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c62d lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.985151:0:19895:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c62d lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764771.985158:0:19895:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764771.985163:0:19895:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970b53100 x1862022966971136/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4624/135824 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764771.985169:0:19895:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c62d lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764771.985184:0:19895:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970b53100 pname:cluuid:pid:xid:nid:opc:job ps:3f83546c-01a9-4156-8a97-44c446245d3e:19895:1862022966971136:192.168.202.146@tcp:101:ps.0 00000100:00100000:1.0:1775764771.985317:0:19895:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f997130d6c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764771.992441:0:19895:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970b53100 pname:cluuid:pid:xid:nid:opc:job ps:3f83546c-01a9-4156-8a97-44c446245d3e:19895:1862022966971136:192.168.202.146@tcp:101:ps.0 00010000:00010000:1.0:1775764771.992465:0:19895:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c62d lrc: 4/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.992480:0:19895:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c62d lrc: 4/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.992487:0:19895:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c62d lrc: 4/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.992494:0:19895:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c62d lrc: 3/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.992502:0:19895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c62d lrc: 4/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.992512:0:19895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970640000 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:1.0:1775764771.992517:0:19895:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c62d lrc: 4/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.992524:0:19895:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c62d lrc: 4/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.992531:0:19895:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c62d lrc: 5/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.992537:0:19895:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1775764771.992544:0:19895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c62d lrc: 3/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.992550:0:19895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c62d lrc: 0/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764771.992569:0:19895:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=7, status=0 req@ffff8f9970b53100 x1862022966971136/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4624/608 e 0 to 0 dl 1775764787 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ps.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764771.992583:0:19895:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0x261:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1775764771.992624:0:19895:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:1.0:1775764771.992636:0:19895:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00010000:1.0:1775764771.992644:0:19895:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c5d9 from it ffffafbac1f37998 00010000:00010000:1.0:1775764771.992647:0:19895:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:1.0:1775764771.992659:0:19895:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00000080:00200000:1.0:1775764771.994176:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.994189:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=glibc-hwcaps, flags=17 00000080:00200000:1.0:1775764771.994198:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.994202:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1775764771.994208:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.994211:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1775764771.994217:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.994219:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1775764771.994226:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.994229:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1775764771.994235:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.994238:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:1.0:1775764771.994246:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.994248:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:1.0:1775764771.994255:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.994258:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:1.0:1775764771.994265:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.994275:0:19895:0:(namei.c:1399:ll_atomic_open()) VFS Op:name=libcap.so.2, dir=[0x200000401:0x261:0x0](ffff8f9952e61988), file ffff8f99613abb00, open_flags 8000, mode 0 opened 0 00000080:00200000:1.0:1775764771.994281:0:19895:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=libcap.so.2, dir=[0x200000401:0x261:0x0](ffff8f9952e61988), intent=open 00000002:00010000:1.0:1775764771.994302:0:19895:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: libcap.so.2,[0x0:0x0:0x0]) in obj [0x200000401:0x261:0x0], intent: open flags 0100001 00010000:00010000:1.0:1775764771.994357:0:19895:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c634 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.994366:0:19895:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c634 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764771.994378:0:19895:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764771.994387:0:19895:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970b53100 x1862022966971392/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4624/135824 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764771.994396:0:19895:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c634 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764771.994417:0:19895:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970b53100 pname:cluuid:pid:xid:nid:opc:job ps:3f83546c-01a9-4156-8a97-44c446245d3e:19895:1862022966971392:192.168.202.146@tcp:101:ps.0 00000100:00100000:1.0:1775764771.994569:0:19895:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f997130d6c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764771.998720:0:19895:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970b53100 pname:cluuid:pid:xid:nid:opc:job ps:3f83546c-01a9-4156-8a97-44c446245d3e:19895:1862022966971392:192.168.202.146@tcp:101:ps.0 00010000:00010000:1.0:1775764771.998771:0:19895:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c634 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.998780:0:19895:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c634 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.998785:0:19895:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c634 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.998789:0:19895:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c634 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.998795:0:19895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c634 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.998798:0:19895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970640200 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:1.0:1775764771.998801:0:19895:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c634 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.998809:0:19895:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c634 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.998814:0:19895:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c634 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.998818:0:19895:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1775764771.998822:0:19895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c634 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.998825:0:19895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c634 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764771.998837:0:19895:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=7, status=0 req@ffff8f9970b53100 x1862022966971392/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4624/608 e 0 to 0 dl 1775764787 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ps.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764771.998848:0:19895:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0x261:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1775764771.999186:0:19895:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00010000:00010000:1.0:1775764771.999197:0:19895:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00000080:00010000:1.0:1775764771.999204:0:19895:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c5d9 from it ffffafbac1f37998 00010000:00010000:1.0:1775764771.999207:0:19895:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00010000:00010000:1.0:1775764771.999212:0:19895:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00000080:00200000:1.0:1775764771.999511:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.999520:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=glibc-hwcaps, flags=17 00000080:00200000:1.0:1775764771.999536:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.999540:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1775764771.999549:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.999552:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1775764771.999560:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.999563:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1775764771.999571:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.999573:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:1.0:1775764771.999578:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.999580:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:1.0:1775764771.999585:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.999587:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:1.0:1775764771.999591:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.999593:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:1.0:1775764771.999597:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764771.999603:0:19895:0:(namei.c:1399:ll_atomic_open()) VFS Op:name=libmount.so.1, dir=[0x200000401:0x261:0x0](ffff8f9952e61988), file ffff8f99439ca780, open_flags 8000, mode 0 opened 0 00000080:00200000:1.0:1775764771.999620:0:19895:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=libmount.so.1, dir=[0x200000401:0x261:0x0](ffff8f9952e61988), intent=open 00000002:00010000:1.0:1775764771.999638:0:19895:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: libmount.so.1,[0x0:0x0:0x0]) in obj [0x200000401:0x261:0x0], intent: open flags 0100001 00010000:00010000:1.0:1775764771.999680:0:19895:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c63b lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764771.999686:0:19895:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c63b lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764771.999693:0:19895:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764771.999698:0:19895:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970b53100 x1862022966971648/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4624/135824 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764771.999705:0:19895:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c63b lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764771.999719:0:19895:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970b53100 pname:cluuid:pid:xid:nid:opc:job ps:3f83546c-01a9-4156-8a97-44c446245d3e:19895:1862022966971648:192.168.202.146@tcp:101:ps.0 00000100:00100000:1.0:1775764772.000964:0:19895:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f997130d6c0 going to sleep for 10 seconds 00000100:00100000:1.0:1775764772.004225:0:19895:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970b53100 pname:cluuid:pid:xid:nid:opc:job ps:3f83546c-01a9-4156-8a97-44c446245d3e:19895:1862022966971648:192.168.202.146@tcp:101:ps.0 00010000:00010000:1.0:1775764772.004246:0:19895:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c63b lrc: 4/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764772.004258:0:19895:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c63b lrc: 4/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764772.004264:0:19895:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c63b lrc: 4/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764772.004270:0:19895:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c63b lrc: 3/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764772.004278:0:19895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c63b lrc: 4/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764772.004284:0:19895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970640000 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:1.0:1775764772.004289:0:19895:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c63b lrc: 4/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764772.004296:0:19895:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c63b lrc: 4/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764772.004303:0:19895:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c63b lrc: 5/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764772.004311:0:19895:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1775764772.004317:0:19895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c63b lrc: 3/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764772.004390:0:19895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c63b lrc: 0/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764772.004409:0:19895:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=7, status=0 req@ffff8f9970b53100 x1862022966971648/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4624/608 e 0 to 0 dl 1775764787 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'ps.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764772.004423:0:19895:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0x261:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1775764772.004449:0:19895:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00010000:00010000:1.0:1775764772.004462:0:19895:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00000080:00010000:1.0:1775764772.004471:0:19895:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c5d9 from it ffffafbac1f37998 00010000:00010000:1.0:1775764772.004475:0:19895:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00010000:00010000:1.0:1775764772.004483:0:19895:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00000080:00200000:0.0:1775764772.006577:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764772.006593:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=glibc-hwcaps, flags=17 00000080:00200000:0.0:1775764772.006626:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764772.006630:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:0.0:1775764772.006639:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764772.006642:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:0.0:1775764772.006649:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764772.006652:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:0.0:1775764772.006659:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764772.006662:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:0.0:1775764772.006669:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764772.006672:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:0.0:1775764772.006680:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764772.006682:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:0.0:1775764772.006688:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764772.006691:0:19895:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:0.0:1775764772.006698:0:19895:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764772.006709:0:19895:0:(namei.c:1399:ll_atomic_open()) VFS Op:name=libgcrypt.so.20, dir=[0x200000401:0x261:0x0](ffff8f9952e61988), file ffff8f994311be00, open_flags 8000, mode 0 opened 0 00000080:00200000:0.0:1775764772.006716:0:19895:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=libgcrypt.so.20, dir=[0x200000401:0x261:0x0](ffff8f9952e61988), intent=open 00000002:00010000:0.0:1775764772.006807:0:19895:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: libgcrypt.so.20,[0x0:0x0:0x0]) in obj [0x200000401:0x261:0x0], intent: open flags 0100001 00010000:00010000:0.0:1775764772.007166:0:19895:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd600/0x36f9bdc9e421c642 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764772.007180:0:19895:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd600/0x36f9bdc9e421c642 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1775764772.007193:0:19895:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1775764772.007205:0:19895:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f0c9180 x1862022966971904/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4624/135824 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1775764772.007213:0:19895:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd600/0x36f9bdc9e421c642 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19895 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764772.007239:0:19895:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c9180 pname:cluuid:pid:xid:nid:opc:job ps:3f83546c-01a9-4156-8a97-44c446245d3e:19895:1862022966971904:192.168.202.146@tcp:101:ps.0 00000100:00100000:0.0:1775764772.009462:0:19895:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f600 going to sleep for 11 seconds 00000080:00200000:1.0:1775764772.127882:0:19698:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:0.0:1775764772.128118:0:19698:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764772.128184:0:19698:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 19698 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764772.128194:0:19698:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 19698 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764772.128203:0:19698:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 19698 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764772.128228:0:19698:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c9180 pname:cluuid:pid:xid:nid:opc:job bash:3f83546c-01a9-4156-8a97-44c446245d3e:19698:1862022966973824:192.168.202.146@tcp:101:bash.0 00000100:00100000:0.0:1775764772.128450:0:19698:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16fb40 going to sleep for 11 seconds 00000100:00100000:0.0:1775764772.138391:0:19698:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c9180 pname:cluuid:pid:xid:nid:opc:job bash:3f83546c-01a9-4156-8a97-44c446245d3e:19698:1862022966973824:192.168.202.146@tcp:101:bash.0 00010000:00010000:0.0:1775764772.138415:0:19698:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764772.138432:0:19698:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764772.138443:0:19698:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764772.138453:0:19698:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764772.138464:0:19698:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764772.138472:0:19698:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764772.138479:0:19698:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764772.138489:0:19698:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994f0c9180 x1862022966973824/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764788 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1775764772.138501:0:19698:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764772.138512:0:19698:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1775764772.138518:0:19698:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1775764772.138543:0:19698:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 8704 00000080:00010000:0.0:1775764772.138550:0:19698:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c67a 00000080:00010000:0.0:1775764772.138555:0:19698:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:0.0:1775764772.138558:0:19698:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c67a from it ffffafbac19c7b58 00010000:00010000:0.0:1775764772.138562:0:19698:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764772.138570:0:19698:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764772.138605:0:19698:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:0.0:1775764772.138642:0:19698:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=d8.sanity-pfl, dir=[0x200000007:0x1:0x0](ffff8f9952d121c8), flags=9 00000080:00200000:0.0:1775764772.138646:0:19698:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=d8.sanity-pfl, dir=[0x200000007:0x1:0x0](ffff8f9952d121c8), intent=getattr 00000002:00010000:0.0:1775764772.138667:0:19698:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: d8.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764772.138705:0:19698:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc000/0x36f9bdc9e421c681 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19698 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764772.138712:0:19698:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc000/0x36f9bdc9e421c681 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19698 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764772.138721:0:19698:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc000/0x36f9bdc9e421c681 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19698 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764772.138773:0:19698:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c9180 pname:cluuid:pid:xid:nid:opc:job bash:3f83546c-01a9-4156-8a97-44c446245d3e:19698:1862022966973952:192.168.202.146@tcp:101:bash.0 00000100:00100000:0.0:1775764772.138873:0:19698:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16fb40 going to sleep for 11 seconds 00010000:00010000:3.0:1775764772.148508:0:11138:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1775764772.148516:0:11138:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc000/0x36f9bdc9e421c681 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5fa888 expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.148524:0:11138:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99821cc000 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:3.0:1775764772.148529:0:11138:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc000/0x36f9bdc9e421c681 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5fa888 expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.148537:0:11138:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc000/0x36f9bdc9e421c681 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x2aac6a50e5fa888 expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.148545:0:11138:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc000/0x36f9bdc9e421c681 lrc: 3/0,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fa888 expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.148560:0:11138:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 30 lru_type: 1 00010000:00010000:3.0:1775764772.148595:0:11138:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc000/0x36f9bdc9e421c681 lrc: 2/0,0 mode: --/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fa888 expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.148601:0:11138:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764772.148635:0:11138:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc000/0x36f9bdc9e421c681 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fa888 expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.148642:0:11138:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc000/0x36f9bdc9e421c681 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x13 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fa888 expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.148655:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764772.148657:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764772.148659:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764772.148674:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970680000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966974080:192.168.202.146@tcp:103:ldlm_bl.0 00000080:00200000:0.0:1775764772.165086:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764772.165100:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=glibc-hwcaps, flags=17 00000080:00200000:0.0:1775764772.165110:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764772.165114:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:0.0:1775764772.165123:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764772.165125:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:0.0:1775764772.165132:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764772.165135:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:0.0:1775764772.165142:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764772.165145:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=tls, flags=17 00000080:00200000:0.0:1775764772.165151:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764772.165155:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:0.0:1775764772.165162:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764772.165164:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:0.0:1775764772.165172:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764772.165174:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=x86_64, flags=17 00000080:00200000:0.0:1775764772.165181:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764772.165184:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=libc.so.6, flags=257 00000080:00200000:0.0:1775764772.165918:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764772.165932:0:19896:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=clients, dir=[0x200000401:0x261:0x0](ffff8f9952e61988), flags=0 00000080:00200000:0.0:1775764772.165936:0:19896:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=clients, dir=[0x200000401:0x261:0x0](ffff8f9952e61988), intent=getattr 00000002:00010000:0.0:1775764772.165959:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: clients,[0x0:0x0:0x0]) in obj [0x200000401:0x261:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764772.166015:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd800/0x36f9bdc9e421c68f lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764772.166026:0:19896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd800/0x36f9bdc9e421c68f lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764772.166035:0:19896:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd800/0x36f9bdc9e421c68f lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764772.166060:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c8380 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966974336:192.168.202.146@tcp:101:rm.0 00000100:00100000:0.0:1775764772.166291:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16fb40 going to sleep for 11 seconds 00000100:00100000:3.0:1775764772.167049:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970680000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966974080:192.168.202.146@tcp:103:ldlm_bl.0 00000080:00200000:0.0:1775764772.206253:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764772.206263:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=258 00000080:00200000:0.0:1775764772.206268:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 44 00000080:00200008:0.0:1775764772.206275:0:19896:0:(file.c:1057:ll_file_open()) START file client0:[0x200000401:0x26a:0x0](ffff8f9952da00c8), flags 704400 00000002:00010000:0.0:1775764772.206294:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x26a:0x0]) in obj [0x200000401:0x268:0x0], intent: open flags 0100240704401 00010000:00010000:0.0:1775764772.206338:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c6ab lrc: 3/1,0 mode: --/CR res: [0x200000401:0x268:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764772.206346:0:19896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c6ab lrc: 3/1,0 mode: --/CR res: [0x200000401:0x268:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1775764772.206356:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1775764772.206362:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f0cb100 x1862022966975616/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/135824 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1775764772.206382:0:19896:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cda00/0x36f9bdc9e421c6ab lrc: 3/1,0 mode: --/CR res: [0x200000401:0x268:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764772.206402:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0cb100 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966975616:192.168.202.146@tcp:101:rm.0 00000100:00100000:0.0:1775764772.206555:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f600 going to sleep for 11 seconds 00000002:00100000:3.0:1775764772.210169:0:19896:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f994f0cb100 x1862022966975616/t4294997532(4294997532) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764788 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200008:3.0:1775764772.210242:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file client0:[0x200000401:0x26a:0x0](ffff8f9952da00c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764772.210249:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8),name=client0 00000002:00010000:3.0:1775764772.210268:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x26a:0x0]) in obj [0x200000401:0x268:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764772.210282:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c6a4 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26a:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa8ea expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764772.210293:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c6a4 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26a:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa8ea expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764772.210299:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x268:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764772.210305:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x26a:0x0](ffff8f9952da00c8) 00000080:00010000:3.0:1775764772.210308:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x26a:0x0](ffff8f9952da00c8) for lock 0x36f9bdc9e421c6a4 00000080:00010000:3.0:1775764772.210313:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c6a4 from it ffffafbac1ec7cd8 00010000:00010000:3.0:1775764772.210316:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c6a4 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26a:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa8ea expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764772.210325:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c6a4 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26a:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa8ea expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764772.210356:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8) pos/size0/7168 32bit_api 0 00000002:00010000:3.0:1775764772.210366:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x26a:0x0]) in obj [0x200000401:0x26a:0x0], intent: readdir flags 0 00010000:00010000:3.0:1775764772.210385:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c6a4 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26a:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa8ea expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764772.210392:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c6a4 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26a:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa8ea expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000100:00100000:3.0:1775764772.211006:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970680000 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966975744:192.168.202.146@tcp:37:rm.0 00000100:00100000:3.0:1775764772.211250:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413d80 going to sleep for 11 seconds 00000100:00100000:3.0:1775764772.229520:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970680000 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966975744:192.168.202.146@tcp:37:rm.0 00010000:00010000:3.0:1775764772.230175:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c6a4 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26a:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa8ea expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764772.230192:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c6a4 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26a:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa8ea expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200008:3.0:1775764772.230250:0:19896:0:(file.c:447:ll_file_release()) START file client0:[0x200000401:0x26a:0x0](ffff8f9952da00c8), flags 704000 00000002:00100000:3.0:1775764772.230313:0:19896:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f994f0cb100 x1862022966975616/t4294997532(4294997532) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764788 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'rm.0' uid:0 gid:0 projid:0 00000020:00100000:3.0:1775764772.230324:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:3.0:1775764772.230330:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970680000 x1862022966976000/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:3.0:1775764772.230351:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970680000 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966976000:192.168.202.146@tcp:35:rm.0 00000100:00100000:3.0:1775764772.233317:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413d80 going to sleep for 11 seconds 00000100:00100000:3.0:1775764772.238696:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970680000 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966976000:192.168.202.146@tcp:35:rm.0 00000002:00100000:3.0:1775764772.238732:0:19896:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f994f0cb100 x1862022966975616/t4294997532(4294997532) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764788 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1775764772.238838:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764772.238848:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=258 00000080:00200000:3.0:1775764772.238855:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 44 00000080:00200008:3.0:1775764772.238863:0:19896:0:(file.c:1057:ll_file_open()) START file client0:[0x200000401:0x26a:0x0](ffff8f9952da00c8), flags 704400 00000002:00010000:3.0:1775764772.238885:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x26a:0x0]) in obj [0x200000401:0x268:0x0], intent: open flags 0104240704401 00010000:00010000:3.0:1775764772.238971:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c6b2 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x268:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.238981:0:19896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c6b2 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x268:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:3.0:1775764772.238992:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1775764772.238998:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970680000 x1862022966976256/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/135824 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1775764772.239008:0:19896:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c6b2 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x268:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764772.239033:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970680000 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966976256:192.168.202.146@tcp:101:rm.0 00000100:00100000:3.0:1775764772.239198:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413d80 going to sleep for 11 seconds 00000100:00100000:3.0:1775764772.245317:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970680000 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966976256:192.168.202.146@tcp:101:rm.0 00010000:00010000:3.0:1775764772.245352:0:19896:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c6b2 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x26a:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa906 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.245364:0:19896:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c6b2 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x26a:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa906 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.245386:0:19896:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c6b2 lrc: 4/1,0 mode: CR/CR res: [0x200000401:0x26a:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa906 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.245396:0:19896:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c6b2 lrc: 4/1,0 mode: CR/CR res: [0x200000401:0x26a:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa906 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764772.245415:0:19896:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=200002b, status=0 req@ffff8f9970680000 x1862022966976256/t4294997534(4294997534) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764788 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00100000:3.0:1775764772.245427:0:19896:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f9970680000 x1862022966976256/t4294997534(4294997534) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764788 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1775764772.245437:0:19896:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c6b2 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0x26a:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa906 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.245447:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 16 res 8589935617/618 (0 0) 00000080:00200000:3.0:1775764772.245455:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0x268:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1775764772.245477:0:19896:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x26a:0x0], updating i_size 7168 00000080:00010000:3.0:1775764772.245482:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x26a:0x0](ffff8f9952da00c8) for lock 0x36f9bdc9e421c6b2 00000080:00010000:3.0:1775764772.245500:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c6b2 from it ffffafbac1ec7bd0 00010000:00010000:3.0:1775764772.245504:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c6b2 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0x26a:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa906 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.245514:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c6b2 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x26a:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa906 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000002:00100000:3.0:1775764772.245539:0:19896:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f9970680000 x1862022966976256/t4294997534(4294997534) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764788 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200008:3.0:1775764772.245620:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file client0:[0x200000401:0x26a:0x0](ffff8f9952da00c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764772.245626:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8),name=client0 00000002:00010000:3.0:1775764772.245648:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x26a:0x0]) in obj [0x200000401:0x268:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764772.245658:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c6a4 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26a:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa8ea expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764772.245671:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c6a4 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26a:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa8ea expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764772.245677:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x268:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764772.245683:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x26a:0x0](ffff8f9952da00c8) 00000080:00010000:3.0:1775764772.245687:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x26a:0x0](ffff8f9952da00c8) for lock 0x36f9bdc9e421c6a4 00000080:00010000:3.0:1775764772.245690:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c6a4 from it ffffafbac1ec7cd8 00010000:00010000:3.0:1775764772.245693:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c6a4 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26a:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa8ea expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764772.245700:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c6a4 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26a:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa8ea expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1775764772.245731:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8) pos/size0/7168 32bit_api 0 00000080:00200000:3.0:1775764772.245794:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8) pos/size18446744073709551614/7168 32bit_api 0 00000080:00200000:3.0:1775764772.245815:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764772.245824:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=0 00000080:00200008:3.0:1775764772.245830:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file ~dmtmp:[0x200000401:0x26c:0x0](ffff8f9952db6c08), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764772.245834:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08),name=~dmtmp 00000002:00010000:3.0:1775764772.245841:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x26c:0x0]) in obj [0x200000401:0x26a:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764772.245850:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764772.245859:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764772.245870:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26a:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764772.245874:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x26c:0x0](ffff8f9952db6c08) 00000080:00010000:3.0:1775764772.245876:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x26c:0x0](ffff8f9952db6c08) for lock 0x36f9bdc9e41cb2b1 00000080:00010000:3.0:1775764772.245879:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e41cb2b1 from it ffffafbac1ec7cb8 00010000:00010000:3.0:1775764772.245882:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764772.245889:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764772.245928:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764772.245933:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=258 00000080:00200000:3.0:1775764772.245937:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 44 00000080:00200008:3.0:1775764772.245945:0:19896:0:(file.c:1057:ll_file_open()) START file ~dmtmp:[0x200000401:0x26c:0x0](ffff8f9952db6c08), flags 704400 00000080:00200008:3.0:1775764772.245958:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file ~dmtmp:[0x200000401:0x26c:0x0](ffff8f9952db6c08), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764772.245962:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08),name=~dmtmp 00000002:00010000:3.0:1775764772.245968:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x26c:0x0]) in obj [0x200000401:0x26a:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764772.245975:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764772.245983:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764772.245990:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26a:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764772.245994:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x26c:0x0](ffff8f9952db6c08) 00000080:00010000:3.0:1775764772.245997:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x26c:0x0](ffff8f9952db6c08) for lock 0x36f9bdc9e41cb2b1 00000080:00010000:3.0:1775764772.246001:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e41cb2b1 from it ffffafbac1ec7cd8 00010000:00010000:3.0:1775764772.246004:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764772.246012:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764772.246025:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08) pos/size0/7168 32bit_api 0 00000002:00010000:3.0:1775764772.246033:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x26c:0x0]) in obj [0x200000401:0x26c:0x0], intent: readdir flags 0 00010000:00010000:3.0:1775764772.246038:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764772.246045:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000100:00100000:3.0:1775764772.246579:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970681c00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966976384:192.168.202.146@tcp:37:rm.0 00000100:00100000:3.0:1775764772.246678:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413d80 going to sleep for 11 seconds 00000100:00100000:3.0:1775764772.260163:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970681c00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966976384:192.168.202.146@tcp:37:rm.0 00010000:00010000:3.0:1775764772.261251:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764772.261266:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200008:3.0:1775764772.261355:0:19896:0:(file.c:447:ll_file_release()) START file ~dmtmp:[0x200000401:0x26c:0x0](ffff8f9952db6c08), flags 704000 00010000:00010000:3.0:1775764772.261379:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971078400/0x36f9bdc9e41ca50c lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ed7b9 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764772.261427:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764772.261436:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=258 00000080:00200000:3.0:1775764772.261443:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 44 00000080:00200008:3.0:1775764772.261451:0:19896:0:(file.c:1057:ll_file_open()) START file ~dmtmp:[0x200000401:0x26c:0x0](ffff8f9952db6c08), flags 704400 00000080:00200008:3.0:1775764772.261468:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file ~dmtmp:[0x200000401:0x26c:0x0](ffff8f9952db6c08), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764772.261473:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08),name=~dmtmp 00000002:00010000:3.0:1775764772.261491:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x26c:0x0]) in obj [0x200000401:0x26a:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764772.261501:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764772.261510:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764772.261518:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26a:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764772.261524:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x26c:0x0](ffff8f9952db6c08) 00000080:00010000:3.0:1775764772.261527:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x26c:0x0](ffff8f9952db6c08) for lock 0x36f9bdc9e41cb2b1 00000080:00010000:3.0:1775764772.261532:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e41cb2b1 from it ffffafbac1ec7cd8 00010000:00010000:3.0:1775764772.261535:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764772.261543:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764772.261568:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08) pos/size0/7168 32bit_api 0 00000080:00200000:3.0:1775764772.261591:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08) pos/size18446744073709551614/7168 32bit_api 0 00000080:00200000:3.0:1775764772.261601:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764772.262108:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=ACCESS, flags=0 00000080:00200008:3.0:1775764772.263127:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file ACCESS:[0x200000401:0x2e6:0x0](ffff8f9952d921c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764772.263138:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2e6:0x0](ffff8f9952d921c8),name=ACCESS 00000002:00010000:3.0:1775764772.263159:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2e6:0x0]) in obj [0x200000401:0x26c:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764772.263172:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/742 (0 0) 00010000:00010000:3.0:1775764772.263223:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6b9 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.263232:0:19896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6b9 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.263242:0:19896:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6b9 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764772.263269:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970681c00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966976512:192.168.202.146@tcp:101:rm.0 00000100:00100000:3.0:1775764772.263403:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413d80 going to sleep for 11 seconds 00000100:00100000:3.0:1775764772.268347:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970681c00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966976512:192.168.202.146@tcp:101:rm.0 00010000:00010000:3.0:1775764772.268378:0:19896:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6b9 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa91b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.268396:0:19896:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6b9 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2e6:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa91b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.268405:0:19896:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6b9 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2e6:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa91b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.268418:0:19896:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6b9 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2e6:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa91b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.268434:0:19896:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6b9 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2e6:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa91b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.268446:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6b9 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2e6:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa91b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.268455:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6b9 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2e6:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa91b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.268462:0:19896:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6b9 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2e6:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa91b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764772.268474:0:19896:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970681c00 x1862022966976512/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/584 e 0 to 0 dl 1775764788 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1775764772.268496:0:19896:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6b9 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2e6:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa91b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.268507:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/742 (0 0) 00000080:00200000:3.0:1775764772.268513:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26c:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1775764772.268533:0:19896:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x2e6:0x0], updating i_size 7168 00000080:00010000:3.0:1775764772.268538:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2e6:0x0](ffff8f9952d921c8) for lock 0x36f9bdc9e421c6b9 00000080:00010000:3.0:1775764772.268548:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2e6:0x0](ffff8f9952d921c8) 00000080:00010000:3.0:1775764772.268551:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c6b9 from it ffffafbac1ec7cb8 00010000:00010000:3.0:1775764772.268554:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6b9 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2e6:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa91b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.268568:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6b9 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e6:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa91b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764772.268654:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764772.268663:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=ACCESS, flags=258 00000080:00200000:3.0:1775764772.268668:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2e6:0x0](ffff8f9952d921c8), inode mode 41c0 mask 44 00000080:00200008:3.0:1775764772.268677:0:19896:0:(file.c:1057:ll_file_open()) START file ACCESS:[0x200000401:0x2e6:0x0](ffff8f9952d921c8), flags 704400 00000080:00200008:3.0:1775764772.268695:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file ACCESS:[0x200000401:0x2e6:0x0](ffff8f9952d921c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764772.268699:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2e6:0x0](ffff8f9952d921c8),name=ACCESS 00000002:00010000:3.0:1775764772.268715:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2e6:0x0]) in obj [0x200000401:0x26c:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764772.268724:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6b9 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2e6:0x0].0x0 bits 0x13/0x0 rrc: 6 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa91b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764772.268734:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6b9 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2e6:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa91b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764772.268783:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26c:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764772.268787:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2e6:0x0](ffff8f9952d921c8) 00000080:00010000:3.0:1775764772.268791:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2e6:0x0](ffff8f9952d921c8) for lock 0x36f9bdc9e421c6b9 00000080:00010000:3.0:1775764772.268795:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c6b9 from it ffffafbac1ec7cd8 00010000:00010000:3.0:1775764772.268798:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6b9 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2e6:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa91b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764772.268805:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6b9 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e6:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa91b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764772.268833:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2e6:0x0](ffff8f9952d921c8) pos/size0/7168 32bit_api 0 00000002:00010000:3.0:1775764772.268844:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2e6:0x0]) in obj [0x200000401:0x2e6:0x0], intent: readdir flags 0 00010000:00010000:3.0:1775764772.268850:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b9c00/0x36f9bdc9e421c2c9 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2e6:0x0].0x0 bits 0x2/0x0 rrc: 6 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8dbc expref: -99 pid: 19859 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764772.268858:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b9c00/0x36f9bdc9e421c2c9 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2e6:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8dbc expref: -99 pid: 19859 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000100:00100000:3.0:1775764772.269419:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970681c00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966976640:192.168.202.146@tcp:37:rm.0 00000100:00100000:3.0:1775764772.269516:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413d80 going to sleep for 11 seconds 00000100:00100000:3.0:1775764772.294725:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970681c00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966976640:192.168.202.146@tcp:37:rm.0 00010000:00010000:3.0:1775764772.295433:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b9c00/0x36f9bdc9e421c2c9 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2e6:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5f8dbc expref: -99 pid: 19859 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764772.295448:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b9c00/0x36f9bdc9e421c2c9 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e6:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8dbc expref: -99 pid: 19859 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764772.295501:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2e6:0x0](ffff8f9952d921c8) pos/size18446744073709551614/7168 32bit_api 0 00000080:00200008:3.0:1775764772.295521:0:19896:0:(file.c:447:ll_file_release()) START file ACCESS:[0x200000401:0x2e6:0x0](ffff8f9952d921c8), flags 704000 00010000:00010000:3.0:1775764772.295545:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e41cb137 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2e6:0x0].0x0 bits 0x5/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ef716 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 93 lru_type: 1 00000080:00200000:3.0:1775764772.295575:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764772.295584:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=ACCESS, flags=0 00000080:00200000:3.0:1775764772.295589:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 3 00000080:00200000:3.0:1775764772.295594:0:19896:0:(namei.c:2305:ll_rmdir()) VFS Op:name=ACCESS, dir=[0x200000401:0x26c:0x0](ffff8f9952db6c08) 00010000:00010000:3.0:1775764772.295633:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764772.295647:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971078400/0x36f9bdc9e41ca50c lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ed7b9 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764772.295783:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6b9 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e6:0x0].0x0 bits 0x13/0x0 rrc: 6 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fa91b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764772.295821:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b9c00/0x36f9bdc9e421c2c9 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e6:0x0].0x0 bits 0x2/0x0 rrc: 6 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f8dbc expref: -99 pid: 19859 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764772.295832:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d48000/0x36f9bdc9e41cf6d5 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e6:0x0].0x0 bits 0x11/0x0 rrc: 6 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e4fd19e expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00010000:00010000:3.0:1775764772.295842:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e41cb137 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2e6:0x0].0x0 bits 0x5/0x0 rrc: 6 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e4ef716 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 93 lru_type: 1 00000002:00100000:3.0:1775764772.295909:0:19896:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f9970f69880 x1862022950427648/t4294969762(4294969762) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764076 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000020:00100000:3.0:1775764772.295920:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:3.0:1775764772.295925:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970681c00 x1862022966976896/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:3.0:1775764772.295945:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970681c00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966976896:192.168.202.146@tcp:35:rm.0 00000100:00100000:3.0:1775764772.296045:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413d80 going to sleep for 11 seconds 00000100:00100000:3.0:1775764772.309201:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970681c00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966976896:192.168.202.146@tcp:35:rm.0 00000002:00100000:3.0:1775764772.309238:0:19896:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f9970f69880 x1862022950427648/t4294969762(4294969762) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764076 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00010000:00010000:3.0:1775764772.309358:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6b9 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2e6:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fa91b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764772.309369:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b9c00/0x36f9bdc9e421c2c9 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2e6:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8dbc expref: -99 pid: 19859 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764772.309387:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d48000/0x36f9bdc9e41cf6d5 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2e6:0x0].0x0 bits 0x11/0x0 rrc: 5 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4fd19e expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00010000:00010000:3.0:1775764772.309393:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e41cb137 lrc: 1/0,0 mode: --/CR res: [0x200000401:0x2e6:0x0].0x0 bits 0x5/0x0 rrc: 5 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ef716 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 93 lru_type: 1 00010000:00010000:3.0:1775764772.309400:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764772.309406:0:19896:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 5 locks packed 00010000:00010000:3.0:1775764772.309408:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6b9 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2e6:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fa91b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764772.309423:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b9c00/0x36f9bdc9e421c2c9 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2e6:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8dbc expref: -99 pid: 19859 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764772.309432:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9966d48000/0x36f9bdc9e41cf6d5 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2e6:0x0].0x0 bits 0x11/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4fd19e expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00010000:00010000:3.0:1775764772.309440:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e41cb137 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x2e6:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ef716 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 93 lru_type: 1 00010000:00010000:3.0:1775764772.309459:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a45e00/0x36f9bdc9e41cb2b1 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4efb22 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000020:00100000:3.0:1775764772.309476:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1775764772.309482:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970681c00 x1862022966977152/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 512/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764772.309504:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970681c00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966977152:192.168.202.146@tcp:36:rm.0 00000100:00100000:3.0:1775764772.309585:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413d80 going to sleep for 11 seconds 00000100:00100000:1.0:1775764772.315716:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970681c00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966977152:192.168.202.146@tcp:36:rm.0 00000080:00200000:1.0:1775764772.315791:0:19896:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2e6:0x0](ffff8f9952d921c8) 00000080:00200000:1.0:1775764772.316117:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:1.0:1775764772.316126:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=COREL, flags=0 00000080:00200008:1.0:1775764772.316135:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file COREL:[0x200000401:0x291:0x0](ffff8f9952dccb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764772.316140:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x291:0x0](ffff8f9952dccb08),name=COREL 00000002:00010000:1.0:1775764772.316159:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x291:0x0]) in obj [0x200000401:0x26c:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764772.316171:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/657 (0 0) 00010000:00010000:1.0:1775764772.316222:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c6c0 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764772.316230:0:19896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c6c0 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764772.316240:0:19896:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c6c0 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764772.316261:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970b53100 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966977280:192.168.202.146@tcp:101:rm.0 00000100:00100000:1.0:1775764772.316364:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f997130d6c0 going to sleep for 11 seconds 00000100:00100000:3.0:1775764772.321520:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970b53100 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966977280:192.168.202.146@tcp:101:rm.0 00010000:00010000:3.0:1775764772.321545:0:19896:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c6c0 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa93e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.321560:0:19896:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c6c0 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x291:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa93e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.321567:0:19896:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c6c0 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa93e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.321579:0:19896:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c6c0 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa93e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.321587:0:19896:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c6c0 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa93e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.321599:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c6c0 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa93e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.321621:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c6c0 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa93e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.321628:0:19896:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c6c0 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa93e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764772.321638:0:19896:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970b53100 x1862022966977280/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/584 e 0 to 0 dl 1775764788 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1775764772.321649:0:19896:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c6c0 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa93e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.321658:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/657 (0 0) 00000080:00200000:3.0:1775764772.321664:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26c:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1775764772.321683:0:19896:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x291:0x0], updating i_size 7168 00000080:00010000:3.0:1775764772.321688:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x291:0x0](ffff8f9952dccb08) for lock 0x36f9bdc9e421c6c0 00000080:00010000:3.0:1775764772.321697:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x291:0x0](ffff8f9952dccb08) 00000080:00010000:3.0:1775764772.321700:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c6c0 from it ffffafbac1ec7cb8 00010000:00010000:3.0:1775764772.321703:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c6c0 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa93e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.321712:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c6c0 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa93e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764772.321833:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764772.321842:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=COREL, flags=258 00000080:00200000:3.0:1775764772.321847:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x291:0x0](ffff8f9952dccb08), inode mode 41c0 mask 44 00000080:00200008:3.0:1775764772.321855:0:19896:0:(file.c:1057:ll_file_open()) START file COREL:[0x200000401:0x291:0x0](ffff8f9952dccb08), flags 704400 00000080:00200008:3.0:1775764772.321873:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file COREL:[0x200000401:0x291:0x0](ffff8f9952dccb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764772.321877:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x291:0x0](ffff8f9952dccb08),name=COREL 00000002:00010000:3.0:1775764772.321894:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x291:0x0]) in obj [0x200000401:0x26c:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764772.321914:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c6c0 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa93e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764772.321925:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c6c0 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa93e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764772.321937:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26c:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764772.321941:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x291:0x0](ffff8f9952dccb08) 00000080:00010000:3.0:1775764772.321944:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x291:0x0](ffff8f9952dccb08) for lock 0x36f9bdc9e421c6c0 00000080:00010000:3.0:1775764772.321948:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c6c0 from it ffffafbac1ec7cd8 00010000:00010000:3.0:1775764772.321950:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c6c0 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa93e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764772.321957:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c6c0 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa93e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764772.322023:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x291:0x0](ffff8f9952dccb08) pos/size0/7168 32bit_api 0 00000002:00010000:3.0:1775764772.322035:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x291:0x0]) in obj [0x200000401:0x291:0x0], intent: readdir flags 0 00010000:00010000:3.0:1775764772.322041:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c6c0 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa93e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764772.322048:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c6c0 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa93e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000100:00100000:3.0:1775764772.322624:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970f69880 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966977408:192.168.202.146@tcp:37:rm.0 00000100:00100000:3.0:1775764772.322710:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413e40 going to sleep for 11 seconds 00000100:00100000:3.0:1775764772.336238:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970f69880 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966977408:192.168.202.146@tcp:37:rm.0 00010000:00010000:3.0:1775764772.337543:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c6c0 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa93e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764772.337561:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c6c0 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa93e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764772.337662:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x291:0x0](ffff8f9952dccb08) pos/size18446744073709551614/7168 32bit_api 0 00000080:00200008:3.0:1775764772.337681:0:19896:0:(file.c:447:ll_file_release()) START file COREL:[0x200000401:0x291:0x0](ffff8f9952dccb08), flags 704000 00010000:00010000:3.0:1775764772.337709:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970fd7200/0x36f9bdc9e41ca567 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x291:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ed8e6 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 93 lru_type: 1 00000080:00200000:3.0:1775764772.337784:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764772.337795:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=COREL, flags=0 00000080:00200000:3.0:1775764772.337802:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 3 00000080:00200000:3.0:1775764772.337807:0:19896:0:(namei.c:2305:ll_rmdir()) VFS Op:name=COREL, dir=[0x200000401:0x26c:0x0](ffff8f9952db6c08) 00010000:00010000:3.0:1775764772.337833:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c6c0 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fa93e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764772.339026:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99700e4000/0x36f9bdc9e41ca59f lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x291:0x0].0x0 bits 0x11/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e4ed980 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00010000:00010000:3.0:1775764772.339054:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970fd7200/0x36f9bdc9e41ca567 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x291:0x0].0x0 bits 0x5/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e4ed8e6 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 93 lru_type: 1 00000002:00100000:3.0:1775764772.339152:0:19896:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f9970ede300 x1862022950312448/t4294969570(4294969570) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764069 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000020:00100000:3.0:1775764772.339164:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:3.0:1775764772.339169:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970f69880 x1862022966977664/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:3.0:1775764772.339192:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970f69880 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966977664:192.168.202.146@tcp:35:rm.0 00000100:00100000:3.0:1775764772.339348:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413e40 going to sleep for 11 seconds 00000100:00100000:3.0:1775764772.350153:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970f69880 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966977664:192.168.202.146@tcp:35:rm.0 00000002:00100000:3.0:1775764772.350189:0:19896:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f9970ede300 x1862022950312448/t4294969570(4294969570) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764069 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00010000:00010000:3.0:1775764772.350308:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c6c0 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fa93e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764772.350329:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99700e4000/0x36f9bdc9e41ca59f lrc: 1/0,0 mode: --/PR res: [0x200000401:0x291:0x0].0x0 bits 0x11/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ed980 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00010000:00010000:3.0:1775764772.350336:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970fd7200/0x36f9bdc9e41ca567 lrc: 1/0,0 mode: --/CR res: [0x200000401:0x291:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ed8e6 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 93 lru_type: 1 00010000:00010000:3.0:1775764772.350342:0:19896:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:3.0:1775764772.350345:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c6c0 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x291:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fa93e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764772.350356:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99700e4000/0x36f9bdc9e41ca59f lrc: 0/0,0 mode: --/PR res: [0x200000401:0x291:0x0].0x0 bits 0x11/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ed980 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00010000:00010000:3.0:1775764772.350378:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970fd7200/0x36f9bdc9e41ca567 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x291:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ed8e6 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 93 lru_type: 1 00000020:00100000:3.0:1775764772.350399:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1775764772.350404:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970f69880 x1862022966977920/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764772.350425:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970f69880 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966977920:192.168.202.146@tcp:36:rm.0 00000100:00100000:3.0:1775764772.350502:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413e40 going to sleep for 11 seconds 00000100:00100000:3.0:1775764772.366953:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970f69880 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966977920:192.168.202.146@tcp:36:rm.0 00000080:00200000:3.0:1775764772.366999:0:19896:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x291:0x0](ffff8f9952dccb08) 00000080:00200000:3.0:1775764772.367086:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764772.367096:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=WORDPRO, flags=0 00000080:00200008:3.0:1775764772.367105:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file WORDPRO:[0x200000401:0x2aa:0x0](ffff8f9952d12a08), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764772.367111:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2aa:0x0](ffff8f9952d12a08),name=WORDPRO 00000002:00010000:3.0:1775764772.367129:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2aa:0x0]) in obj [0x200000401:0x26c:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764772.367141:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/682 (0 0) 00010000:00010000:3.0:1775764772.367191:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c6c7 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.367200:0:19896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c6c7 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.367210:0:19896:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c6c7 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764772.367229:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966978048:192.168.202.146@tcp:101:rm.0 00000100:00100000:3.0:1775764772.367301:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413e40 going to sleep for 11 seconds 00010000:00010000:1.0:1775764772.368035:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764772.372480:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966978048:192.168.202.146@tcp:101:rm.0 00010000:00010000:3.0:1775764772.372505:0:19896:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c6c7 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa961 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.372524:0:19896:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c6c7 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2aa:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa961 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.372531:0:19896:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c6c7 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2aa:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa961 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.372543:0:19896:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c6c7 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2aa:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa961 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.372553:0:19896:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c6c7 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2aa:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa961 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.372565:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c6c7 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2aa:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa961 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.372574:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c6c7 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2aa:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa961 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.372581:0:19896:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c6c7 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2aa:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa961 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764772.372592:0:19896:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970ede300 x1862022966978048/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/584 e 0 to 0 dl 1775764788 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1775764772.372616:0:19896:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c6c7 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2aa:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa961 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.372626:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/682 (0 0) 00000080:00200000:3.0:1775764772.372632:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26c:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1775764772.372652:0:19896:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x2aa:0x0], updating i_size 7168 00000080:00010000:3.0:1775764772.372658:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2aa:0x0](ffff8f9952d12a08) for lock 0x36f9bdc9e421c6c7 00000080:00010000:3.0:1775764772.372667:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2aa:0x0](ffff8f9952d12a08) 00000080:00010000:3.0:1775764772.372670:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c6c7 from it ffffafbac1ec7cb8 00010000:00010000:3.0:1775764772.372673:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c6c7 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2aa:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa961 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.372680:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c6c7 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2aa:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa961 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764772.372775:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764772.372783:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=WORDPRO, flags=258 00000080:00200000:3.0:1775764772.372789:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2aa:0x0](ffff8f9952d12a08), inode mode 41c0 mask 44 00000080:00200008:3.0:1775764772.372798:0:19896:0:(file.c:1057:ll_file_open()) START file WORDPRO:[0x200000401:0x2aa:0x0](ffff8f9952d12a08), flags 704400 00000080:00200008:3.0:1775764772.372822:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file WORDPRO:[0x200000401:0x2aa:0x0](ffff8f9952d12a08), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764772.372828:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2aa:0x0](ffff8f9952d12a08),name=WORDPRO 00000002:00010000:3.0:1775764772.372847:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2aa:0x0]) in obj [0x200000401:0x26c:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764772.372857:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c6c7 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2aa:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa961 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764772.372868:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c6c7 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2aa:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa961 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764772.372880:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26c:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764772.372884:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2aa:0x0](ffff8f9952d12a08) 00000080:00010000:3.0:1775764772.372887:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2aa:0x0](ffff8f9952d12a08) for lock 0x36f9bdc9e421c6c7 00000080:00010000:3.0:1775764772.372891:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c6c7 from it ffffafbac1ec7cd8 00010000:00010000:3.0:1775764772.372894:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c6c7 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2aa:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa961 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764772.372901:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c6c7 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2aa:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa961 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764772.372938:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2aa:0x0](ffff8f9952d12a08) pos/size0/7168 32bit_api 0 00000002:00010000:3.0:1775764772.372950:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2aa:0x0]) in obj [0x200000401:0x2aa:0x0], intent: readdir flags 0 00010000:00010000:3.0:1775764772.372956:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c6c7 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2aa:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa961 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764772.372964:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c6c7 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2aa:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa961 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000100:00100000:3.0:1775764772.373583:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966978176:192.168.202.146@tcp:37:rm.0 00000100:00100000:0.0:1775764772.389198:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413e40 going to sleep for 11 seconds 00000100:00100000:1.0:1775764772.401353:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966978176:192.168.202.146@tcp:37:rm.0 00010000:00010000:1.0:1775764772.402422:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c6c7 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2aa:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa961 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764772.402437:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c6c7 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2aa:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa961 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1775764772.463349:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102f00 going to sleep for 11 seconds 00010000:00010000:2.0:1775764772.497032:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9940030000/0x36f9bdc9e41ca2ce lrc: 1/0,0 mode: --/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x11/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ed1f4 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 83 lru_type: 1 00010000:00010000:2.0:1775764772.497039:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971071e00/0x36f9bdc9e41ca29d lrc: 1/0,0 mode: --/CR res: [0x200000401:0x27f:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ed137 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 232 lru_type: 1 00010000:00010000:2.0:1775764772.497044:0:19896:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:2.0:1775764772.497047:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eac00/0x36f9bdc9e421c6ce lrc: 0/0,0 mode: --/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fa984 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764772.497058:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9940030000/0x36f9bdc9e41ca2ce lrc: 0/0,0 mode: --/PR res: [0x200000401:0x27f:0x0].0x0 bits 0x11/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ed1f4 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 83 lru_type: 1 00010000:00010000:2.0:1775764772.497066:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971071e00/0x36f9bdc9e41ca29d lrc: 0/0,0 mode: --/CR res: [0x200000401:0x27f:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ed137 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 232 lru_type: 1 00000020:00100000:2.0:1775764772.497086:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1775764772.497092:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994efdfb80 x1862022966979456/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764772.497112:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdfb80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966979456:192.168.202.146@tcp:36:rm.0 00000100:00100000:2.0:1775764772.497202:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102f00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764772.508205:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdfb80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966979456:192.168.202.146@tcp:36:rm.0 00000080:00200000:2.0:1775764772.508249:0:19896:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x27f:0x0](ffff8f9952db5348) 00000080:00200000:2.0:1775764772.508343:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:2.0:1775764772.508353:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PM, flags=0 00000080:00200008:2.0:1775764772.508364:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file PM:[0x200000401:0x2b2:0x0](ffff8f9952d8ba88), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764772.508370:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2b2:0x0](ffff8f9952d8ba88),name=PM 00000002:00010000:2.0:1775764772.508396:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2b2:0x0]) in obj [0x200000401:0x26c:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764772.508411:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/690 (0 0) 00010000:00010000:2.0:1775764772.508461:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eac00/0x36f9bdc9e421c6d5 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764772.508469:0:19896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eac00/0x36f9bdc9e421c6d5 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764772.508478:0:19896:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eac00/0x36f9bdc9e421c6d5 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764772.508510:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdf800 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966979584:192.168.202.146@tcp:101:rm.0 00000100:00100000:2.0:1775764772.508618:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102f00 going to sleep for 11 seconds 00000100:00100000:1.0:1775764772.508868:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764772.508963:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978360320 00000100:00100000:1.0:1775764772.508970:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978360320 00000100:00100000:1.0:1775764772.509020:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24112 00000100:00100000:1.0:1775764772.509029:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970b50380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:12186:x1862022978360320:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764772.509044:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f31f6b33/0x36f9bdc9e421b82d lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x65d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f6c91 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 1 00010000:00010000:1.0:1775764772.509103:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764772.509109:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970b50380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:12186:x1862022978360320:12345-192.168.202.146@tcp:104: Request processed in 81us (243us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764772.509115:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24112 00010000:00010000:2.0:1775764772.509142:0:19800:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764772.509149:0:19800:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f31f6b33/0x36f9bdc9e421b82d lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x65d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f6c91 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 1 00010000:00010000:2.0:1775764772.509158:0:19800:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f997093c400 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:2.0:1775764772.509163:0:19800:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f31f6b33/0x36f9bdc9e421b82d lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x65d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e5f6c91 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 1 00010000:00010000:1.0:1775764772.509356:0:19800:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f31f6b33/0x36f9bdc9e421b82d lrc: 2/0,0 mode: --/PW res: [0x280000403:0x65d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f6c91 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 1 00010000:00010000:1.0:1775764772.509364:0:19800:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764772.509389:0:19800:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f31f6b33/0x36f9bdc9e421b82d lrc: 1/0,0 mode: --/PW res: [0x280000403:0x65d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f6c91 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 1 00010000:00010000:1.0:1775764772.509398:0:19800:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f31f6b33/0x36f9bdc9e421b82d lrc: 0/0,0 mode: --/PW res: [0x280000403:0x65d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f6c91 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 1 00010000:00010000:1.0:1775764772.509416:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764772.509418:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764772.509420:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764772.509427:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1775764772.509437:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970b51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966979712:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764772.518725:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.518794:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.518816:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.518832:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.518849:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.518865:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.518883:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.518910:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.518947:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.518983:0:11131:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1775764772.519007:0:11131:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978361344 00000100:00100000:0.0:1775764772.519016:0:11131:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978361344 00000100:00100000:0.0:1775764772.519054:0:11131:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978361472 00000100:00100000:0.0:1775764772.519057:0:11131:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978361472 00000100:00100000:0.0:1775764772.519066:0:11131:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978361600 00000100:00100000:0.0:1775764772.519068:0:11131:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978361600 00000100:00100000:0.0:1775764772.519077:0:11131:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978361728 00000100:00100000:0.0:1775764772.519079:0:11131:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978361728 00000100:00100000:0.0:1775764772.519088:0:11131:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978362880 00000100:00100000:0.0:1775764772.519090:0:11131:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978362880 00000100:00100000:0.0:1775764772.519097:0:11131:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978363008 00000100:00100000:0.0:1775764772.519099:0:11131:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978363008 00000100:00100000:0.0:1775764772.519112:0:11131:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24113 00000100:00100000:0.0:1775764772.519121:0:11131:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c9c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+9:12039:x1862022978361344:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764772.519139:0:11131:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006d77473f/0x36f9bdc9e421ba9c lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x65e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f73fa expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 1 00010000:00010000:0.0:1775764772.519199:0:11131:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764772.519205:0:11131:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c9c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+9:12039:x1862022978361344:12345-192.168.202.146@tcp:104: Request processed in 85us (491us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764772.519212:0:11131:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24113 00000100:00100000:0.0:1775764772.519228:0:11131:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24114 00000100:00100000:0.0:1775764772.519234:0:11131:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c9f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+8:12040:x1862022978361472:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764772.519243:0:11131:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000000c000ce/0x36f9bdc9e41cb1d1 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x6a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef8d6 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764772.519268:0:11131:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:0.0:1775764772.519273:0:11131:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c9f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+8:12040:x1862022978361472:12345-192.168.202.146@tcp:104: Request processed in 40us (479us total) trans 0 rc 0/0 00000100:00100000:3.0:1775764772.519460:0:5563:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:3.0:1775764772.519471:0:5563:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978363136 00000100:00100000:3.0:1775764772.519474:0:5563:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978363136 00000100:00100000:3.0:1775764772.519493:0:5563:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978363264 00000100:00100000:3.0:1775764772.519495:0:5563:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978363264 00000100:00100000:3.0:1775764772.519505:0:5563:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978363392 00000100:00100000:3.0:1775764772.519507:0:5563:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978363392 00000100:00100000:3.0:1775764772.519519:0:5563:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 1102 00000100:00100000:3.0:1775764772.519525:0:5563:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0cbb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+9:12051:x1862022978363136:12345-192.168.202.146@tcp:104: 00010000:00010000:3.0:1775764772.519534:0:5563:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000003c3894b8/0x36f9bdc9e421c213 lrc: 2/0,0 mode: PR/PR res: [0x280000403:0x34:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5f8a2e expref: -99 pid: 19852 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764772.519565:0:5563:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:3.0:1775764772.519570:0:5563:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0cbb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+9:12051:x1862022978363136:12345-192.168.202.146@tcp:104: Request processed in 45us (686us total) trans 0 rc 0/0 00000100:00100000:3.0:1775764772.519576:0:5563:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 1102 00000100:00100000:2.0:1775764772.519581:0:11142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 1103 00000100:00100000:2.0:1775764772.519589:0:11142:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0cb100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+5:12041:x1862022978363264:12345-192.168.202.146@tcp:104: 00000100:00100000:3.0:1775764772.519589:0:5563:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 1104 00000100:00100000:3.0:1775764772.519593:0:5563:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f13a450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+5:8216:x1862022978363392:12345-192.168.202.146@tcp:104: 00010000:00010000:2.0:1775764772.519597:0:11142:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000dd363438/0x36f9bdc9e421b6d6 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x65d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f687e expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 1 00010000:00010000:3.0:1775764772.519599:0:5563:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000121a9b09/0x36f9bdc9e421bcb7 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f7a91 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 1 00010000:00010000:3.0:1775764772.519640:0:5563:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/5 locks to priority blp list, 5 blwis in pool 00000100:00100000:3.0:1775764772.519704:0:5563:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f13a450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+5:8216:x1862022978363392:12345-192.168.202.146@tcp:104: Request processed in 110us (757us total) trans 0 rc 0/0 00000100:00100000:3.0:1775764772.519711:0:5563:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 1104 00000100:00100000:0.0:1775764772.519761:0:11131:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24114 00010000:00010000:2.0:1775764772.523662:0:11142:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/5 locks to priority blp list, 5 blwis in pool 00000100:00100000:2.0:1775764772.523686:0:11142:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0cb100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:12041:x1862022978363264:12345-192.168.202.146@tcp:104: Request processed in 4088us (4765us total) trans 0 rc 0/0 00000100:00100000:2.0:1775764772.523704:0:11142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 1103 00000100:00100000:1.0:1775764772.525927:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24115 00010000:00010000:1.0:1775764772.525954:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 4 total in blp. (4 blwis in pool) 00010000:00010000:1.0:1775764772.525963:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006d77473f/0x36f9bdc9e421ba9c lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x65e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f73fa expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 1 00010000:00010000:1.0:1775764772.525983:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9941481600 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:1.0:1775764772.525990:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006d77473f/0x36f9bdc9e421ba9c lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x65e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e5f73fa expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 1 00010000:00010000:1.0:1775764772.526096:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006d77473f/0x36f9bdc9e421ba9c lrc: 2/0,0 mode: --/PW res: [0x280000403:0x65e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f73fa expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 1 00010000:00010000:1.0:1775764772.526104:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764772.526139:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006d77473f/0x36f9bdc9e421ba9c lrc: 1/0,0 mode: --/PW res: [0x280000403:0x65e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f73fa expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 1 00010000:00010000:1.0:1775764772.526147:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006d77473f/0x36f9bdc9e421ba9c lrc: 0/0,0 mode: --/PW res: [0x280000403:0x65e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f73fa expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 1 00010000:00010000:1.0:1775764772.526169:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 3 total in blp. (3 blwis in pool) 00010000:00010000:1.0:1775764772.526173:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000000c000ce/0x36f9bdc9e41cb1d1 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x6a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef8d6 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764772.526180:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970a44c00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:1.0:1775764772.526184:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000000c000ce/0x36f9bdc9e41cb1d1 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x6a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ef8d6 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764772.526210:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000000c000ce/0x36f9bdc9e41cb1d1 lrc: 2/0,0 mode: --/PW res: [0x280000403:0x6a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef8d6 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764772.526222:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764772.526227:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000000c000ce/0x36f9bdc9e41cb1d1 lrc: 1/0,0 mode: --/PW res: [0x280000403:0x6a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef8d6 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764772.526234:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000000c000ce/0x36f9bdc9e41cb1d1 lrc: 0/0,0 mode: --/PW res: [0x280000403:0x6a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef8d6 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764772.526245:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:1.0:1775764772.526249:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000003c3894b8/0x36f9bdc9e421c213 lrc: 2/0,0 mode: PR/PR res: [0x280000403:0x34:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5f8a2e expref: -99 pid: 19852 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764772.526255:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99668b8800 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:1.0:1775764772.526259:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000003c3894b8/0x36f9bdc9e421c213 lrc: 3/0,0 mode: PR/PR res: [0x280000403:0x34:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x428400010000 nid: local remote: 0x2aac6a50e5f8a2e expref: -99 pid: 19852 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764772.526292:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000003c3894b8/0x36f9bdc9e421c213 lrc: 2/0,0 mode: --/PR res: [0x280000403:0x34:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5f8a2e expref: -99 pid: 19852 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764772.526298:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764772.526302:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000003c3894b8/0x36f9bdc9e421c213 lrc: 1/0,0 mode: --/PR res: [0x280000403:0x34:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5f8a2e expref: -99 pid: 19852 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764772.526309:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000003c3894b8/0x36f9bdc9e421c213 lrc: 0/0,0 mode: --/PR res: [0x280000403:0x34:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5f8a2e expref: -99 pid: 19852 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764772.526321:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1775764772.526326:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000121a9b09/0x36f9bdc9e421bcb7 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f7a91 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 1 00010000:00010000:1.0:1775764772.526333:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f994ed0ce00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:1.0:1775764772.526336:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000121a9b09/0x36f9bdc9e421bcb7 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e5f7a91 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 1 00010000:00010000:1.0:1775764772.526362:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000121a9b09/0x36f9bdc9e421bcb7 lrc: 2/0,0 mode: --/PW res: [0x240000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f7a91 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 1 00010000:00010000:1.0:1775764772.526369:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764772.526372:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000121a9b09/0x36f9bdc9e421bcb7 lrc: 1/0,0 mode: --/PW res: [0x240000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f7a91 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 1 00010000:00010000:1.0:1775764772.526380:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000121a9b09/0x36f9bdc9e421bcb7 lrc: 0/0,0 mode: --/PW res: [0x240000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f7a91 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 1 00010000:00010000:1.0:1775764772.526392:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764772.526395:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000dd363438/0x36f9bdc9e421b6d6 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x65d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f687e expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 1 00010000:00010000:1.0:1775764772.526402:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99668bee00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:1.0:1775764772.526405:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000dd363438/0x36f9bdc9e421b6d6 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x65d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e5f687e expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 1 00010000:00010000:1.0:1775764772.526427:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000dd363438/0x36f9bdc9e421b6d6 lrc: 2/0,0 mode: --/PW res: [0x240000403:0x65d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f687e expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 1 00010000:00010000:1.0:1775764772.526439:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764772.526442:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000dd363438/0x36f9bdc9e421b6d6 lrc: 1/0,0 mode: --/PW res: [0x240000403:0x65d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f687e expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 1 00010000:00010000:1.0:1775764772.526448:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000dd363438/0x36f9bdc9e421b6d6 lrc: 0/0,0 mode: --/PW res: [0x240000403:0x65d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f687e expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 1 00010000:00010000:1.0:1775764772.526459:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764772.526460:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764772.526462:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764772.526469:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764772.526474:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764772.526491:0:11146:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24116 00000100:00100000:1.0:1775764772.526498:0:11146:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0ca680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+7:12184:x1862022978361728:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764772.526508:0:11146:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000048e87692/0x36f9bdc9e41cb129 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x68:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef6f3 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764772.526583:0:11146:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764772.526590:0:11146:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0ca680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+7:12184:x1862022978361728:12345-192.168.202.146@tcp:104: Request processed in 93us (7757us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764772.526597:0:11146:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24116 00000100:00100000:1.0:1775764772.526626:0:11146:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24117 00000100:00100000:1.0:1775764772.526633:0:11146:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0cad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+6:12187:x1862022978362880:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764772.526641:0:11146:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000ff7ec1b5/0x36f9bdc9e421c1bf lrc: 2/0,0 mode: PR/PR res: [0x280000403:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5f8971 expref: -99 pid: 19850 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764772.526671:0:11146:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1775764772.526675:0:11146:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0cad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+6:12187:x1862022978362880:12345-192.168.202.146@tcp:104: Request processed in 43us (7826us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764772.526681:0:11146:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24117 00000100:00100000:1.0:1775764772.526691:0:11146:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24118 00000100:00100000:1.0:1775764772.526696:0:11146:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970e0a450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+5:12193:x1862022978363008:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764772.526703:0:11146:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000043cd0e4b/0x36f9bdc9e41cb184 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x69:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef7e1 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764772.526723:0:11146:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:1.0:1775764772.526727:0:11146:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970e0a450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+5:12193:x1862022978363008:12345-192.168.202.146@tcp:104: Request processed in 31us (7862us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764772.526733:0:11146:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24118 00010000:00010000:1.0:1775764772.526768:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:1.0:1775764772.526772:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000048e87692/0x36f9bdc9e41cb129 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x68:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef6f3 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764772.526779:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9966d49600 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:1.0:1775764772.526783:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000048e87692/0x36f9bdc9e41cb129 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x68:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ef6f3 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764772.526815:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000048e87692/0x36f9bdc9e41cb129 lrc: 2/0,0 mode: --/PW res: [0x280000403:0x68:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef6f3 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764772.526822:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764772.526827:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000048e87692/0x36f9bdc9e41cb129 lrc: 1/0,0 mode: --/PW res: [0x280000403:0x68:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef6f3 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764772.526835:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000048e87692/0x36f9bdc9e41cb129 lrc: 0/0,0 mode: --/PW res: [0x280000403:0x68:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef6f3 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764772.526847:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1775764772.526850:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000ff7ec1b5/0x36f9bdc9e421c1bf lrc: 2/0,0 mode: PR/PR res: [0x280000403:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5f8971 expref: -99 pid: 19850 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764772.526856:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99668b8400 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:1.0:1775764772.526860:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000ff7ec1b5/0x36f9bdc9e421c1bf lrc: 3/0,0 mode: PR/PR res: [0x280000403:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x428400010000 nid: local remote: 0x2aac6a50e5f8971 expref: -99 pid: 19850 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764772.526884:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000ff7ec1b5/0x36f9bdc9e421c1bf lrc: 2/0,0 mode: --/PR res: [0x280000403:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5f8971 expref: -99 pid: 19850 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764772.526889:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764772.526893:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000ff7ec1b5/0x36f9bdc9e421c1bf lrc: 1/0,0 mode: --/PR res: [0x280000403:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5f8971 expref: -99 pid: 19850 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764772.526900:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000ff7ec1b5/0x36f9bdc9e421c1bf lrc: 0/0,0 mode: --/PR res: [0x280000403:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5f8971 expref: -99 pid: 19850 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764772.526910:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764772.526913:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000043cd0e4b/0x36f9bdc9e41cb184 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x69:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef7e1 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764772.526924:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99710c8c00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:1.0:1775764772.526927:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000043cd0e4b/0x36f9bdc9e41cb184 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x69:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ef7e1 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764772.526950:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000043cd0e4b/0x36f9bdc9e41cb184 lrc: 2/0,0 mode: --/PW res: [0x280000403:0x69:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef7e1 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764772.526956:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764772.526959:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000043cd0e4b/0x36f9bdc9e41cb184 lrc: 1/0,0 mode: --/PW res: [0x280000403:0x69:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef7e1 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764772.526967:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000043cd0e4b/0x36f9bdc9e41cb184 lrc: 0/0,0 mode: --/PW res: [0x280000403:0x69:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef7e1 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764772.526978:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764772.526979:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764772.526980:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764772.526984:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764772.526998:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970b50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966979840:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764772.527039:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970b53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966980096:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764772.527052:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970b51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966980352:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764772.527067:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970b50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966980608:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764772.527082:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 4 async RPCs [1->0] 00000100:00100000:1.0:1775764772.527087:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970b53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966979968:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764772.527098:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970b51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966980224:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764772.527110:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970b52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966980480:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764772.527121:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970b51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966980736:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764772.527144:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c9880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:6578:x1862022978361600:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764772.527153:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000003876501d/0x36f9bdc9e421c19c lrc: 2/0,0 mode: PR/PR res: [0x280000403:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5f8955 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764772.527172:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764772.527176:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c9880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:6578:x1862022978361600:12345-192.168.202.146@tcp:104: Request processed in 1240us (8361us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764772.527182:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24115 00010000:00010000:1.0:1775764772.527203:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764772.527207:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000003876501d/0x36f9bdc9e421c19c lrc: 2/0,0 mode: PR/PR res: [0x280000403:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5f8955 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764772.527215:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970924400 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:1.0:1775764772.527218:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000003876501d/0x36f9bdc9e421c19c lrc: 3/0,0 mode: PR/PR res: [0x280000403:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x428400010000 nid: local remote: 0x2aac6a50e5f8955 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764772.527250:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000003876501d/0x36f9bdc9e421c19c lrc: 2/0,0 mode: --/PR res: [0x280000403:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5f8955 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764772.527256:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764772.527265:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000003876501d/0x36f9bdc9e421c19c lrc: 1/0,0 mode: --/PR res: [0x280000403:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5f8955 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764772.527273:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000003876501d/0x36f9bdc9e421c19c lrc: 0/0,0 mode: --/PR res: [0x280000403:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5f8955 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764772.527284:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764772.527285:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764772.527287:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764772.527293:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970b50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966980864:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764772.527382:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764772.527454:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764772.527478:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764772.527512:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764772.527662:0:11146:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978363520 00000100:00100000:1.0:1775764772.527668:0:11146:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978363520 00000100:00100000:1.0:1775764772.527691:0:11146:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978363648 00000100:00100000:1.0:1775764772.527694:0:11146:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978363648 00000100:00100000:1.0:1775764772.527702:0:11146:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978363776 00000100:00100000:1.0:1775764772.527704:0:11146:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978363776 00000100:00100000:1.0:1775764772.527712:0:11146:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978363904 00000100:00100000:1.0:1775764772.527714:0:11146:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978363904 00000100:00100000:1.0:1775764772.527729:0:11146:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24119 00000100:00100000:1.0:1775764772.527734:0:11146:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970b52d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+7:12180:x1862022978363520:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764772.527770:0:11146:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001c16e60e/0x36f9bdc9e41cb15a lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x69:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef78d expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 164 lru_type: 1 00010000:00010000:1.0:1775764772.527801:0:11146:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764772.527807:0:11146:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970b52d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+7:12180:x1862022978363520:12345-192.168.202.146@tcp:104: Request processed in 73us (427us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764772.527813:0:11146:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24119 00000100:00100000:1.0:1775764772.527825:0:11146:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24120 00000100:00100000:1.0:1775764772.527830:0:11146:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970b52680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+6:12190:x1862022978363648:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764772.527838:0:11146:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a0078ea5/0x36f9bdc9e421b976 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x65e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f709d expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 0 00010000:00010000:1.0:1775764772.527861:0:11146:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1775764772.527865:0:11146:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970b52680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+6:12190:x1862022978363648:12345-192.168.202.146@tcp:104: Request processed in 36us (412us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764772.527872:0:11146:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24120 00000100:00100000:1.0:1775764772.527883:0:11146:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24121 00000100:00100000:1.0:1775764772.527888:0:11146:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970b53480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+5:12215:x1862022978363776:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764772.527895:0:11146:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000031510ace/0x36f9bdc9e421c20c lrc: 2/0,0 mode: PR/PR res: [0x240000403:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5f8a27 expref: -99 pid: 19852 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764772.527920:0:11146:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:1.0:1775764772.527925:0:11146:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970b53480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+5:12215:x1862022978363776:12345-192.168.202.146@tcp:104: Request processed in 37us (446us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764772.527930:0:11146:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24121 00000100:00100000:1.0:1775764772.527942:0:11146:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24122 00000100:00100000:1.0:1775764772.527947:0:11146:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0a9c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+4:6579:x1862022978363904:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764772.527954:0:11146:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001e605740/0x36f9bdc9e421c1e2 lrc: 2/0,0 mode: PR/PR res: [0x240000403:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5f89fd expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764772.527980:0:11146:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/4 locks to priority blp list, 4 blwis in pool 00000100:00100000:1.0:1775764772.527984:0:11146:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0a9c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+4:6579:x1862022978363904:12345-192.168.202.146@tcp:104: Request processed in 37us (473us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764772.527990:0:11146:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24122 00010000:00010000:1.0:1775764772.528003:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 3 total in blp. (3 blwis in pool) 00010000:00010000:1.0:1775764772.528007:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001c16e60e/0x36f9bdc9e41cb15a lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x69:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef78d expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 164 lru_type: 1 00010000:00010000:1.0:1775764772.528015:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9966d48c00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:1.0:1775764772.528019:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001c16e60e/0x36f9bdc9e41cb15a lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x69:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ef78d expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 164 lru_type: 1 00010000:00010000:1.0:1775764772.528062:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001c16e60e/0x36f9bdc9e41cb15a lrc: 2/0,0 mode: --/PW res: [0x240000403:0x69:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef78d expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 164 lru_type: 1 00010000:00010000:1.0:1775764772.528070:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764772.528081:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001c16e60e/0x36f9bdc9e41cb15a lrc: 1/0,0 mode: --/PW res: [0x240000403:0x69:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef78d expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 164 lru_type: 1 00010000:00010000:1.0:1775764772.528088:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001c16e60e/0x36f9bdc9e41cb15a lrc: 0/0,0 mode: --/PW res: [0x240000403:0x69:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef78d expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 164 lru_type: 1 00010000:00010000:1.0:1775764772.528102:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:1.0:1775764772.528105:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a0078ea5/0x36f9bdc9e421b976 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x65e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f709d expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 0 00010000:00010000:1.0:1775764772.528115:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9971078800 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:1.0:1775764772.528119:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a0078ea5/0x36f9bdc9e421b976 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x65e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e5f709d expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 0 00010000:00010000:1.0:1775764772.528152:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a0078ea5/0x36f9bdc9e421b976 lrc: 2/0,0 mode: --/PW res: [0x240000403:0x65e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f709d expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 0 00010000:00010000:1.0:1775764772.528159:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764772.528165:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a0078ea5/0x36f9bdc9e421b976 lrc: 1/0,0 mode: --/PW res: [0x240000403:0x65e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f709d expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 0 00010000:00010000:1.0:1775764772.528172:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a0078ea5/0x36f9bdc9e421b976 lrc: 0/0,0 mode: --/PW res: [0x240000403:0x65e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f709d expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 25 lru_type: 0 00010000:00010000:1.0:1775764772.528183:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1775764772.528186:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000031510ace/0x36f9bdc9e421c20c lrc: 2/0,0 mode: PR/PR res: [0x240000403:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5f8a27 expref: -99 pid: 19852 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764772.528192:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99668b9a00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:1.0:1775764772.528195:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000031510ace/0x36f9bdc9e421c20c lrc: 3/0,0 mode: PR/PR res: [0x240000403:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x428400010000 nid: local remote: 0x2aac6a50e5f8a27 expref: -99 pid: 19852 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764772.528225:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000031510ace/0x36f9bdc9e421c20c lrc: 2/0,0 mode: --/PR res: [0x240000403:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5f8a27 expref: -99 pid: 19852 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764772.528238:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764772.528243:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000031510ace/0x36f9bdc9e421c20c lrc: 1/0,0 mode: --/PR res: [0x240000403:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5f8a27 expref: -99 pid: 19852 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764772.528249:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000031510ace/0x36f9bdc9e421c20c lrc: 0/0,0 mode: --/PR res: [0x240000403:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5f8a27 expref: -99 pid: 19852 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764772.528262:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764772.528264:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001e605740/0x36f9bdc9e421c1e2 lrc: 2/0,0 mode: PR/PR res: [0x240000403:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5f89fd expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764772.528270:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f994ed0c600 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:1.0:1775764772.528274:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001e605740/0x36f9bdc9e421c1e2 lrc: 3/0,0 mode: PR/PR res: [0x240000403:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x428400010000 nid: local remote: 0x2aac6a50e5f89fd expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764772.528300:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764772.528304:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001e605740/0x36f9bdc9e421c1e2 lrc: 2/0,0 mode: --/PR res: [0x240000403:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5f89fd expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764772.528310:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764772.528315:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001e605740/0x36f9bdc9e421c1e2 lrc: 1/0,0 mode: --/PR res: [0x240000403:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5f89fd expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764772.528322:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001e605740/0x36f9bdc9e421c1e2 lrc: 0/0,0 mode: --/PR res: [0x240000403:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5f89fd expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764772.528333:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764772.528335:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764772.528337:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764772.528341:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764772.528345:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764772.528355:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0a8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966981120:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764772.528389:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0a8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966981376:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764772.528412:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1775764772.528418:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966980992:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764772.528439:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966981248:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1775764772.528532:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764772.532193:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764772.537211:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764772.537247:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:2.0:1775764772.539184:0:19896:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8f99523ee800: committing for last_committed 4294997538 gen 1 00000100:00100000:2.0:1775764772.539877:0:19896:0:(client.c:3131:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff8f994efde680 x1862022966975232/t4294997531(4294997531) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764788 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'rm.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1775764772.539891:0:19896:0:(client.c:3131:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff8f9970680000 x1862022966976256/t4294997534(4294997534) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764788 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'rm.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1775764772.539918:0:19896:0:(client.c:3126:ptlrpc_free_committed()) @@@ stopping search req@ffff8f994efdd880 x1862022966978688/t4294997540(4294997540) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/456 e 0 to 0 dl 1775764788 ref 1 fl Complete:RQU/204/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764772.540379:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdf800 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966979584:192.168.202.146@tcp:101:rm.0 00010000:00010000:2.0:1775764772.540403:0:19896:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eac00/0x36f9bdc9e421c6d5 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faa17 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764772.540417:0:19896:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eac00/0x36f9bdc9e421c6d5 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faa17 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764772.540423:0:19896:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eac00/0x36f9bdc9e421c6d5 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faa17 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764772.540434:0:19896:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eac00/0x36f9bdc9e421c6d5 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5faa17 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764772.540444:0:19896:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eac00/0x36f9bdc9e421c6d5 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faa17 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764772.540456:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eac00/0x36f9bdc9e421c6d5 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5faa17 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764772.540464:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eac00/0x36f9bdc9e421c6d5 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5faa17 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764772.540471:0:19896:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eac00/0x36f9bdc9e421c6d5 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5faa17 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764772.540588:0:11146:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978364032 00000100:00100000:1.0:1775764772.540597:0:11146:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978364032 00000002:00100000:2.0:1775764772.540657:0:19896:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994efdf800 x1862022966979584/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/584 e 0 to 0 dl 1775764788 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1775764772.540661:0:11146:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978364160 00000100:00100000:1.0:1775764772.540664:0:11146:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978364160 00000002:00010000:2.0:1775764772.540673:0:19896:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eac00/0x36f9bdc9e421c6d5 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faa17 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764772.540679:0:11146:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24123 00010000:00010000:2.0:1775764772.540686:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/690 (0 0) 00000100:00100000:1.0:1775764772.540689:0:11146:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e7480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+5:6580:x1862022978364032:12345-192.168.202.146@tcp:104: 00000080:00200000:2.0:1775764772.540692:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26c:0x0] suppgids 0 0: rc 0 00010000:00010000:1.0:1775764772.540702:0:11146:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000e7187ba4/0x36f9bdc9e421c21a lrc: 2/0,0 mode: PR/PR res: [0x240000403:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5f8a35 expref: -99 pid: 19852 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764772.540713:0:19896:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x2b2:0x0], updating i_size 7168 00000080:00010000:2.0:1775764772.540718:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2b2:0x0](ffff8f9952d8ba88) for lock 0x36f9bdc9e421c6d5 00000080:00010000:2.0:1775764772.540728:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2b2:0x0](ffff8f9952d8ba88) 00000080:00010000:2.0:1775764772.540731:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c6d5 from it ffffafbac1ec7cb8 00010000:00010000:2.0:1775764772.540734:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eac00/0x36f9bdc9e421c6d5 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5faa17 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764772.540762:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eac00/0x36f9bdc9e421c6d5 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faa17 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764772.540838:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:2.0:1775764772.540847:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PM, flags=258 00000080:00200000:2.0:1775764772.540852:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b2:0x0](ffff8f9952d8ba88), inode mode 41c0 mask 44 00000080:00200008:2.0:1775764772.540860:0:19896:0:(file.c:1057:ll_file_open()) START file PM:[0x200000401:0x2b2:0x0](ffff8f9952d8ba88), flags 704400 00000080:00200008:2.0:1775764772.540878:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file PM:[0x200000401:0x2b2:0x0](ffff8f9952d8ba88), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764772.540883:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2b2:0x0](ffff8f9952d8ba88),name=PM 00010000:00010000:1.0:1775764772.541008:0:11146:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764772.541015:0:11146:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e7480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+5:6580:x1862022978364032:12345-192.168.202.146@tcp:104: Request processed in 327us (3819us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764772.541022:0:11146:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24123 00000100:00100000:1.0:1775764772.541040:0:11146:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24124 00000002:00010000:2.0:1775764772.541045:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2b2:0x0]) in obj [0x200000401:0x26c:0x0], intent: getattr flags 0 00000100:00100000:1.0:1775764772.541046:0:11146:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e5c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+4:16465:x1862022978364160:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764772.541056:0:11146:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000bb07259e/0x36f9bdc9e421a67c lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x653:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f37af expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:2.0:1775764772.541056:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eac00/0x36f9bdc9e421c6d5 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faa17 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764772.541067:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eac00/0x36f9bdc9e421c6d5 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faa17 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764772.541074:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26c:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764772.541079:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2b2:0x0](ffff8f9952d8ba88) 00000080:00010000:2.0:1775764772.541082:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2b2:0x0](ffff8f9952d8ba88) for lock 0x36f9bdc9e421c6d5 00000080:00010000:2.0:1775764772.541086:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c6d5 from it ffffafbac1ec7cd8 00010000:00010000:2.0:1775764772.541088:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eac00/0x36f9bdc9e421c6d5 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5faa17 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764772.541096:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eac00/0x36f9bdc9e421c6d5 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faa17 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764772.541115:0:19800:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000080:00200000:2.0:1775764772.541119:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2b2:0x0](ffff8f9952d8ba88) pos/size0/7168 32bit_api 0 00010000:00010000:3.0:1775764772.541122:0:19800:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000e7187ba4/0x36f9bdc9e421c21a lrc: 2/0,0 mode: PR/PR res: [0x240000403:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5f8a35 expref: -99 pid: 19852 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764772.541129:0:19800:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99668b8200 already unused, calling callback (ffffffffc12102c0) 00000002:00010000:2.0:1775764772.541131:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2b2:0x0]) in obj [0x200000401:0x2b2:0x0], intent: readdir flags 0 00010000:00010000:3.0:1775764772.541135:0:19800:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000e7187ba4/0x36f9bdc9e421c21a lrc: 3/0,0 mode: PR/PR res: [0x240000403:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x428400010000 nid: local remote: 0x2aac6a50e5f8a35 expref: -99 pid: 19852 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764772.541137:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eac00/0x36f9bdc9e421c6d5 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faa17 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764772.541145:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eac00/0x36f9bdc9e421c6d5 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faa17 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764772.541213:0:19800:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000e7187ba4/0x36f9bdc9e421c21a lrc: 2/0,0 mode: --/PR res: [0x240000403:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5f8a35 expref: -99 pid: 19852 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764772.541220:0:19800:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764772.541243:0:19800:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000e7187ba4/0x36f9bdc9e421c21a lrc: 1/0,0 mode: --/PR res: [0x240000403:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5f8a35 expref: -99 pid: 19852 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764772.541250:0:19800:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000e7187ba4/0x36f9bdc9e421c21a lrc: 0/0,0 mode: --/PR res: [0x240000403:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5f8a35 expref: -99 pid: 19852 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764772.541266:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764772.541268:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764772.541269:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764772.541283:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966981504:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.545822:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdf800 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966981632:192.168.202.146@tcp:37:rm.0 00000100:00100000:0.0:1775764772.551491:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970b51880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966979712:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1775764772.552467:0:11146:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764772.552480:0:11146:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e5c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+4:16465:x1862022978364160:12345-192.168.202.146@tcp:104: Request processed in 11430us (15229us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764772.552493:0:11146:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24124 00010000:00010000:1.0:1775764772.552521:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764772.552529:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000bb07259e/0x36f9bdc9e421a67c lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x653:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f37af expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764772.552542:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970925000 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:1.0:1775764772.552550:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000bb07259e/0x36f9bdc9e421a67c lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x653:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e5f37af expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764772.552677:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000bb07259e/0x36f9bdc9e421a67c lrc: 2/0,0 mode: --/PW res: [0x240000403:0x653:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f37af expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764772.552685:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764772.552703:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000bb07259e/0x36f9bdc9e421a67c lrc: 1/0,0 mode: --/PW res: [0x240000403:0x653:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f37af expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764772.552711:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000bb07259e/0x36f9bdc9e421a67c lrc: 0/0,0 mode: --/PW res: [0x240000403:0x653:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f37af expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764772.552734:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764772.552764:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764772.552767:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764772.552783:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966981760:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764772.553325:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970b50380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966980864:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764772.553340:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970b51500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966980224:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764772.553350:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970b52a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966980480:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764772.553359:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970b51f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966980736:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764772.553367:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970b50e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966979840:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764772.553377:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970b53100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966980096:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764772.553386:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970b50700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966980608:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764772.581241:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764772.581364:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966981760:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764772.581410:0:5561:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1775764772.581426:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978364416 00000100:00100000:1.0:1775764772.581433:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978364416 00000100:00100000:1.0:1775764772.581477:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24125 00000100:00100000:1.0:1775764772.581486:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9966bf6050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:12215:x1862022978364416:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764772.581505:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000415212e9/0x36f9bdc9e41cab48 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x54:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ee83d expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 34 lru_type: 1 00010000:00010000:1.0:1775764772.581592:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764772.581599:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9966bf6050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:12215:x1862022978364416:12345-192.168.202.146@tcp:104: Request processed in 113us (370us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764772.581618:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24125 00000100:00100000:1.0:1775764772.581691:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0aad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966980992:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764772.581701:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0a8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966981248:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764772.581710:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0a8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966981120:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764772.581717:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0a8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966981376:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764772.581726:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970b53b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966979968:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764772.581734:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970b51180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966980352:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1775764772.582045:0:5572:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1775764772.582052:0:5572:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000415212e9/0x36f9bdc9e41cab48 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x54:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ee83d expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 34 lru_type: 1 00010000:00010000:3.0:1775764772.582062:0:5572:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970640800 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:3.0:1775764772.582068:0:5572:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000415212e9/0x36f9bdc9e41cab48 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x54:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ee83d expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 34 lru_type: 1 00010000:00010000:3.0:1775764772.582152:0:5572:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000415212e9/0x36f9bdc9e41cab48 lrc: 2/0,0 mode: --/PW res: [0x280000403:0x54:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee83d expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 34 lru_type: 1 00010000:00010000:3.0:1775764772.582160:0:5572:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764772.582182:0:5572:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000415212e9/0x36f9bdc9e41cab48 lrc: 1/0,0 mode: --/PW res: [0x280000403:0x54:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee83d expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 34 lru_type: 1 00010000:00010000:3.0:1775764772.582195:0:5572:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000415212e9/0x36f9bdc9e41cab48 lrc: 0/0,0 mode: --/PW res: [0x280000403:0x54:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee83d expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 34 lru_type: 1 00010000:00010000:3.0:1775764772.582216:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764772.582218:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764772.582220:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764772.582247:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966981888:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.582725:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102f00 going to sleep for 11 seconds 00000100:00100000:0.0:1775764772.585188:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.585220:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.585240:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.585258:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.585820:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.585836:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.585854:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.585871:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764772.585986:0:5561:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1775764772.586006:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978365312 00000100:00100000:1.0:1775764772.586015:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978365312 00000100:00100000:1.0:1775764772.586049:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978365440 00000100:00100000:1.0:1775764772.586052:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978365440 00000100:00100000:1.0:1775764772.586060:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978365568 00000100:00100000:1.0:1775764772.586062:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978365568 00000100:00100000:1.0:1775764772.586069:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978365696 00000100:00100000:1.0:1775764772.586071:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978365696 00000100:00100000:1.0:1775764772.586078:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978365824 00000100:00100000:1.0:1775764772.586080:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978365824 00000100:00100000:1.0:1775764772.586086:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978365952 00000100:00100000:1.0:1775764772.586088:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978365952 00000100:00100000:1.0:1775764772.586095:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978366208 00000100:00100000:1.0:1775764772.586097:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978366208 00000100:00100000:1.0:1775764772.586109:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24126 00000100:00100000:1.0:1775764772.586117:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c8380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+7:12193:x1862022978365312:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764772.586134:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000006afc0c20/0x36f9bdc9e41ca66a lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x46:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4edb86 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 42 lru_type: 1 00010000:00010000:1.0:1775764772.586193:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764772.586199:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c8380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+7:12193:x1862022978365312:12345-192.168.202.146@tcp:104: Request processed in 83us (1022us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764772.586206:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24126 00000100:00100000:1.0:1775764772.586220:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24127 00000100:00100000:1.0:1775764772.586225:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c9180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+6:6579:x1862022978365440:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764772.586233:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000002a06534e/0x36f9bdc9e421af04 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x659:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f5101 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 1 00010000:00010000:1.0:1775764772.586263:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1775764772.586268:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c9180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+6:6579:x1862022978365440:12345-192.168.202.146@tcp:104: Request processed in 43us (1048us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764772.586274:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24127 00000100:00100000:1.0:1775764772.586298:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24128 00000100:00100000:1.0:1775764772.586304:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c9500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+6:12039:x1862022978365568:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764772.586311:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a988520c/0x36f9bdc9e41ca598 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x43:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ed964 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 92 lru_type: 1 00000100:00100000:0.0:1775764772.586435:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.586492:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.586505:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.586518:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.586771:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.586792:0:11146:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978367104 00000100:00100000:0.0:1775764772.586797:0:11146:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978367104 00000100:00100000:0.0:1775764772.586816:0:11146:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978367232 00000100:00100000:0.0:1775764772.586819:0:11146:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978367232 00000100:00100000:0.0:1775764772.586827:0:11146:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978367360 00000100:00100000:0.0:1775764772.586829:0:11146:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978367360 00000100:00100000:0.0:1775764772.586837:0:11146:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978367488 00000100:00100000:0.0:1775764772.586839:0:11146:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978367488 00000100:00100000:0.0:1775764772.586846:0:11146:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978367872 00000100:00100000:0.0:1775764772.586847:0:11146:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978367872 00000100:00100000:0.0:1775764772.586857:0:11146:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24129 00000100:00100000:0.0:1775764772.586862:0:11146:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0ca300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+9:12184:x1862022978365696:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764772.586871:0:11146:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000d31a5730/0x36f9bdc9e41ca758 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x49:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eddaf expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 30 lru_type: 1 00010000:00010000:0.0:1775764772.586928:0:11146:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:0.0:1775764772.586933:0:11146:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0ca300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+9:12184:x1862022978365696:12345-192.168.202.146@tcp:104: Request processed in 71us (1675us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764772.586939:0:11146:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24129 00000100:00100000:0.0:1775764772.586953:0:11146:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24130 00000100:00100000:0.0:1775764772.586959:0:11146:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c8000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+7:12040:x1862022978365824:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764772.586967:0:11146:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000071573dcf/0x36f9bdc9e41cabbf lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x56:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ee97f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 0 00010000:00010000:0.0:1775764772.586997:0:11146:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/4 locks to priority blp list, 4 blwis in pool 00000100:00100000:0.0:1775764772.587003:0:11146:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c8000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+7:12040:x1862022978365824:12345-192.168.202.146@tcp:104: Request processed in 44us (1184us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764772.587008:0:11146:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24130 00000100:00100000:0.0:1775764772.587019:0:11146:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24131 00000100:00100000:0.0:1775764772.587025:0:11146:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c8700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+8:12041:x1862022978365952:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764772.587033:0:11146:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000076c55670/0x36f9bdc9e41cab87 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x55:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ee8ec expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 34 lru_type: 1 00010000:00010000:0.0:1775764772.587077:0:11146:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/5 locks to priority blp list, 5 blwis in pool 00000100:00100000:0.0:1775764772.587082:0:11146:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c8700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+8:12041:x1862022978365952:12345-192.168.202.146@tcp:104: Request processed in 57us (1245us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764772.587088:0:11146:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24131 00000100:00100000:0.0:1775764772.587100:0:11146:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24132 00000100:00100000:0.0:1775764772.587106:0:11146:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970e0b050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+6:16465:x1862022978366208:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764772.587114:0:11146:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001ba52963/0x36f9bdc9e41cb011 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x66:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef3dc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 1 00010000:00010000:0.0:1775764772.587141:0:11146:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/6 locks to priority blp list, 6 blwis in pool 00000100:00100000:0.0:1775764772.587146:0:11146:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970e0b050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+6:16465:x1862022978366208:12345-192.168.202.146@tcp:104: Request processed in 40us (1291us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764772.587152:0:11146:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24132 00000100:00100000:0.0:1775764772.587163:0:11146:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24133 00000100:00100000:0.0:1775764772.587170:0:11146:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0cb480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+7:6578:x1862022978367104:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764772.587179:0:11146:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000839c629c/0x36f9bdc9e421a3c0 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x651:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f2e5c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 0 00010000:00010000:0.0:1775764772.587202:0:11146:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/7 locks to priority blp list, 7 blwis in pool 00000100:00100000:0.0:1775764772.587207:0:11146:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0cb480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+7:6578:x1862022978367104:12345-192.168.202.146@tcp:104: Request processed in 38us (773us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764772.587216:0:11146:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24133 00000100:00100000:0.0:1775764772.587226:0:11146:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24134 00000100:00100000:0.0:1775764772.587231:0:11146:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c8a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+6:8216:x1862022978367232:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764772.587239:0:11146:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000063e812bc/0x36f9bdc9e4210eba lrc: 2/0,0 mode: PR/PR res: [0x280000403:0x4a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5d3d47 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00010000:00010000:0.0:1775764772.587260:0:11146:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/8 locks to priority blp list, 8 blwis in pool 00000100:00100000:0.0:1775764772.587264:0:11146:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c8a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+6:8216:x1862022978367232:12345-192.168.202.146@tcp:104: Request processed in 33us (773us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764772.587269:0:11146:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24134 00000100:00100000:0.0:1775764772.587280:0:11146:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24135 00000100:00100000:0.0:1775764772.587285:0:11146:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0caa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+5:12196:x1862022978367360:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764772.587293:0:11146:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000e02763ec/0x36f9bdc9e41ca82a lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x4c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4edfa7 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 1 00010000:00010000:0.0:1775764772.587313:0:11146:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/9 locks to priority blp list, 9 blwis in pool 00010000:00010000:1.0:1775764772.587691:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/10 locks to priority blp list, 10 blwis in pool 00000100:00100000:1.0:1775764772.587697:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c9500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:12039:x1862022978365568:12345-192.168.202.146@tcp:104: Request processed in 1393us (2456us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764772.587704:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24128 00000100:00100000:1.0:1775764772.587731:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24136 00000100:00100000:1.0:1775764772.587755:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c8e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:12190:x1862022978367488:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764772.587764:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f22973fd/0x36f9bdc9e41ca5ec lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x44:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eda44 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 93 lru_type: 1 00010000:00010000:1.0:1775764772.587786:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/11 locks to priority blp list, 11 blwis in pool 00000100:00100000:1.0:1775764772.587790:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c8e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:12190:x1862022978367488:12345-192.168.202.146@tcp:104: Request processed in 53us (1272us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764772.587797:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24136 00000100:00100000:1.0:1775764772.587809:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24137 00000100:00100000:1.0:1775764772.587814:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c9c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:12182:x1862022978367872:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764772.587823:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000083f2c617/0x36f9bdc9e41caaf4 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x54:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ee756 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 69 lru_type: 1 00010000:00010000:1.0:1775764772.587842:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/12 locks to priority blp list, 12 blwis in pool 00000100:00100000:1.0:1775764772.587847:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c9c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:12182:x1862022978367872:12345-192.168.202.146@tcp:104: Request processed in 33us (1077us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764772.587852:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24137 00010000:00010000:1.0:1775764772.587869:0:11210:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 11 total in blp. (11 blwis in pool) 00010000:00010000:1.0:1775764772.587875:0:11210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000006afc0c20/0x36f9bdc9e41ca66a lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x46:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4edb86 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 42 lru_type: 1 00010000:00010000:1.0:1775764772.587882:0:11210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970fd7c00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:1.0:1775764772.587888:0:11210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000006afc0c20/0x36f9bdc9e41ca66a lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x46:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4edb86 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 42 lru_type: 1 00010000:00010000:1.0:1775764772.587971:0:11210:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000006afc0c20/0x36f9bdc9e41ca66a lrc: 2/0,0 mode: --/PW res: [0x240000403:0x46:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4edb86 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 42 lru_type: 1 00010000:00010000:1.0:1775764772.587979:0:11210:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764772.588004:0:11210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000006afc0c20/0x36f9bdc9e41ca66a lrc: 1/0,0 mode: --/PW res: [0x240000403:0x46:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4edb86 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 42 lru_type: 1 00010000:00010000:1.0:1775764772.588012:0:11210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000006afc0c20/0x36f9bdc9e41ca66a lrc: 0/0,0 mode: --/PW res: [0x240000403:0x46:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4edb86 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 42 lru_type: 1 00010000:00010000:1.0:1775764772.588032:0:11210:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 10 total in blp. (10 blwis in pool) 00010000:00010000:1.0:1775764772.588036:0:11210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000002a06534e/0x36f9bdc9e421af04 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x659:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f5101 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 1 00010000:00010000:1.0:1775764772.588043:0:11210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99668bf400 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:1.0:1775764772.588059:0:11210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000002a06534e/0x36f9bdc9e421af04 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x659:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e5f5101 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 1 00010000:00010000:1.0:1775764772.588093:0:11210:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000002a06534e/0x36f9bdc9e421af04 lrc: 2/0,0 mode: --/PW res: [0x280000403:0x659:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f5101 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 1 00010000:00010000:1.0:1775764772.588101:0:11210:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764772.588107:0:11210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000002a06534e/0x36f9bdc9e421af04 lrc: 1/0,0 mode: --/PW res: [0x280000403:0x659:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f5101 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 1 00010000:00010000:1.0:1775764772.588115:0:11210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000002a06534e/0x36f9bdc9e421af04 lrc: 0/0,0 mode: --/PW res: [0x280000403:0x659:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f5101 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 1 00010000:00010000:1.0:1775764772.588129:0:11210:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 9 total in blp. (9 blwis in pool) 00010000:00010000:1.0:1775764772.588132:0:11210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000d31a5730/0x36f9bdc9e41ca758 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x49:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eddaf expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 30 lru_type: 1 00010000:00010000:1.0:1775764772.588139:0:11210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99700e5a00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:1.0:1775764772.588142:0:11210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000d31a5730/0x36f9bdc9e41ca758 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x49:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4eddaf expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 30 lru_type: 1 00010000:00010000:1.0:1775764772.588170:0:11210:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000d31a5730/0x36f9bdc9e41ca758 lrc: 2/0,0 mode: --/PW res: [0x280000403:0x49:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eddaf expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 30 lru_type: 1 00010000:00010000:1.0:1775764772.588181:0:11210:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764772.588186:0:11210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000d31a5730/0x36f9bdc9e41ca758 lrc: 1/0,0 mode: --/PW res: [0x280000403:0x49:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eddaf expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 30 lru_type: 1 00010000:00010000:1.0:1775764772.588193:0:11210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000d31a5730/0x36f9bdc9e41ca758 lrc: 0/0,0 mode: --/PW res: [0x280000403:0x49:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eddaf expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 30 lru_type: 1 00010000:00010000:1.0:1775764772.588206:0:11210:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 8 total in blp. (8 blwis in pool) 00010000:00010000:1.0:1775764772.588209:0:11210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000071573dcf/0x36f9bdc9e41cabbf lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x56:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ee97f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 0 00010000:00010000:1.0:1775764772.588215:0:11210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970fd7000 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:1.0:1775764772.588218:0:11210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000071573dcf/0x36f9bdc9e41cabbf lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x56:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ee97f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 0 00010000:00010000:1.0:1775764772.588246:0:11210:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000071573dcf/0x36f9bdc9e41cabbf lrc: 2/0,0 mode: --/PW res: [0x240000403:0x56:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee97f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 0 00010000:00010000:1.0:1775764772.588252:0:11210:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764772.588257:0:11210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000071573dcf/0x36f9bdc9e41cabbf lrc: 1/0,0 mode: --/PW res: [0x240000403:0x56:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee97f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 0 00010000:00010000:1.0:1775764772.588264:0:11210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000071573dcf/0x36f9bdc9e41cabbf lrc: 0/0,0 mode: --/PW res: [0x240000403:0x56:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee97f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 0 00010000:00010000:1.0:1775764772.588275:0:11210:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 7 total in blp. (7 blwis in pool) 00010000:00010000:1.0:1775764772.588279:0:11210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000076c55670/0x36f9bdc9e41cab87 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x55:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ee8ec expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 34 lru_type: 1 00010000:00010000:1.0:1775764772.588300:0:11210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99700e5600 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:1.0:1775764772.588303:0:11210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000076c55670/0x36f9bdc9e41cab87 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x55:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ee8ec expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 34 lru_type: 1 00000100:00100000:0.0:1775764772.589273:0:11146:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0caa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+4:12196:x1862022978367360:12345-192.168.202.146@tcp:104: Request processed in 33us (812us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764772.590205:0:11146:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24135 00010000:00010000:2.0:1775764772.591618:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 6 total in blp. (6 blwis in pool) 00010000:00010000:2.0:1775764772.591630:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001ba52963/0x36f9bdc9e41cb011 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x66:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef3dc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 1 00010000:00010000:2.0:1775764772.591645:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f994ed0cc00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:2.0:1775764772.591652:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001ba52963/0x36f9bdc9e41cb011 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x66:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ef3dc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 1 00010000:00010000:2.0:1775764772.591723:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001ba52963/0x36f9bdc9e41cb011 lrc: 2/0,0 mode: --/PW res: [0x240000403:0x66:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef3dc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 1 00010000:00010000:1.0:1775764772.591724:0:11210:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000076c55670/0x36f9bdc9e41cab87 lrc: 2/0,0 mode: --/PW res: [0x280000403:0x55:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee8ec expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 34 lru_type: 1 00010000:00010000:2.0:1775764772.591731:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764772.591732:0:11210:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764772.591757:0:11210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000076c55670/0x36f9bdc9e41cab87 lrc: 1/0,0 mode: --/PW res: [0x280000403:0x55:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee8ec expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 34 lru_type: 1 00010000:00010000:1.0:1775764772.591765:0:11210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000076c55670/0x36f9bdc9e41cab87 lrc: 0/0,0 mode: --/PW res: [0x280000403:0x55:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee8ec expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 34 lru_type: 1 00010000:00010000:2.0:1775764772.591776:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001ba52963/0x36f9bdc9e41cb011 lrc: 1/0,0 mode: --/PW res: [0x240000403:0x66:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef3dc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 1 00010000:00010000:2.0:1775764772.591783:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001ba52963/0x36f9bdc9e41cb011 lrc: 0/0,0 mode: --/PW res: [0x240000403:0x66:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef3dc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 1 00010000:00010000:1.0:1775764772.591785:0:11210:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 5 total in blp. (5 blwis in pool) 00010000:00010000:1.0:1775764772.591789:0:11210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000839c629c/0x36f9bdc9e421a3c0 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x651:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f2e5c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 0 00010000:00010000:1.0:1775764772.591795:0:11210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99702fc800 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:2.0:1775764772.591796:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 4 total in blp. (4 blwis in pool) 00010000:00010000:1.0:1775764772.591799:0:11210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000839c629c/0x36f9bdc9e421a3c0 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x651:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e5f2e5c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 0 00010000:00010000:2.0:1775764772.591800:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000063e812bc/0x36f9bdc9e4210eba lrc: 2/0,0 mode: PR/PR res: [0x280000403:0x4a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5d3d47 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00010000:00010000:2.0:1775764772.591807:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970f6c000 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:2.0:1775764772.591811:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000063e812bc/0x36f9bdc9e4210eba lrc: 3/0,0 mode: PR/PR res: [0x280000403:0x4a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x428400010000 nid: local remote: 0x2aac6a50e5d3d47 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00010000:00010000:1.0:1775764772.591834:0:11210:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000839c629c/0x36f9bdc9e421a3c0 lrc: 2/0,0 mode: --/PW res: [0x280000403:0x651:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f2e5c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 0 00010000:00010000:2.0:1775764772.591836:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000063e812bc/0x36f9bdc9e4210eba lrc: 2/0,0 mode: --/PR res: [0x280000403:0x4a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5d3d47 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00010000:00010000:2.0:1775764772.591844:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764772.591847:0:11210:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764772.591849:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000063e812bc/0x36f9bdc9e4210eba lrc: 1/0,0 mode: --/PR res: [0x280000403:0x4a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5d3d47 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00010000:00010000:1.0:1775764772.591852:0:11210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000839c629c/0x36f9bdc9e421a3c0 lrc: 1/0,0 mode: --/PW res: [0x280000403:0x651:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f2e5c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 0 00010000:00010000:2.0:1775764772.591857:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000063e812bc/0x36f9bdc9e4210eba lrc: 0/0,0 mode: --/PR res: [0x280000403:0x4a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5d3d47 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00010000:00010000:1.0:1775764772.591860:0:11210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000839c629c/0x36f9bdc9e421a3c0 lrc: 0/0,0 mode: --/PW res: [0x280000403:0x651:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f2e5c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 0 00010000:00010000:2.0:1775764772.591870:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 3 total in blp. (3 blwis in pool) 00010000:00010000:1.0:1775764772.591872:0:11210:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:2.0:1775764772.591874:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000e02763ec/0x36f9bdc9e41ca82a lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x4c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4edfa7 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 1 00010000:00010000:1.0:1775764772.591876:0:11210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a988520c/0x36f9bdc9e41ca598 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x43:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ed964 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 92 lru_type: 1 00010000:00010000:2.0:1775764772.591880:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f996695d200 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:1.0:1775764772.591883:0:11210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9971078000 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:2.0:1775764772.591883:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000e02763ec/0x36f9bdc9e41ca82a lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x4c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4edfa7 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 1 00010000:00010000:1.0:1775764772.591886:0:11210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a988520c/0x36f9bdc9e41ca598 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x43:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ed964 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 92 lru_type: 1 00010000:00010000:2.0:1775764772.591909:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000e02763ec/0x36f9bdc9e41ca82a lrc: 2/0,0 mode: --/PW res: [0x280000403:0x4c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4edfa7 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 1 00010000:00010000:1.0:1775764772.591912:0:11210:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a988520c/0x36f9bdc9e41ca598 lrc: 2/0,0 mode: --/PW res: [0x240000403:0x43:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed964 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 92 lru_type: 1 00010000:00010000:2.0:1775764772.591915:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764772.591919:0:11210:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764772.591919:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000e02763ec/0x36f9bdc9e41ca82a lrc: 1/0,0 mode: --/PW res: [0x280000403:0x4c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4edfa7 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 1 00010000:00010000:1.0:1775764772.591922:0:11210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a988520c/0x36f9bdc9e41ca598 lrc: 1/0,0 mode: --/PW res: [0x240000403:0x43:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed964 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 92 lru_type: 1 00010000:00010000:2.0:1775764772.591931:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000e02763ec/0x36f9bdc9e41ca82a lrc: 0/0,0 mode: --/PW res: [0x280000403:0x4c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4edfa7 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 1 00010000:00010000:1.0:1775764772.591940:0:11210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a988520c/0x36f9bdc9e41ca598 lrc: 0/0,0 mode: --/PW res: [0x240000403:0x43:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed964 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 92 lru_type: 1 00010000:00010000:2.0:1775764772.591944:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1775764772.591947:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f22973fd/0x36f9bdc9e41ca5ec lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x44:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eda44 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 93 lru_type: 1 00010000:00010000:1.0:1775764772.591952:0:11210:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764772.591953:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99700e5800 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:1.0:1775764772.591956:0:11210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000083f2c617/0x36f9bdc9e41caaf4 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x54:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ee756 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 69 lru_type: 1 00010000:00010000:2.0:1775764772.591957:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f22973fd/0x36f9bdc9e41ca5ec lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x44:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4eda44 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 93 lru_type: 1 00010000:00010000:1.0:1775764772.591963:0:11210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970641400 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:1.0:1775764772.591966:0:11210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000083f2c617/0x36f9bdc9e41caaf4 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x54:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ee756 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 69 lru_type: 1 00010000:00010000:2.0:1775764772.591986:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f22973fd/0x36f9bdc9e41ca5ec lrc: 2/0,0 mode: --/PW res: [0x280000403:0x44:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eda44 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 93 lru_type: 1 00010000:00010000:2.0:1775764772.591993:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764772.591996:0:11210:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000083f2c617/0x36f9bdc9e41caaf4 lrc: 2/0,0 mode: --/PW res: [0x240000403:0x54:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee756 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 69 lru_type: 1 00010000:00010000:2.0:1775764772.591997:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f22973fd/0x36f9bdc9e41ca5ec lrc: 1/0,0 mode: --/PW res: [0x280000403:0x44:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eda44 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 93 lru_type: 1 00010000:00010000:1.0:1775764772.592003:0:11210:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764772.592005:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f22973fd/0x36f9bdc9e41ca5ec lrc: 0/0,0 mode: --/PW res: [0x280000403:0x44:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eda44 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 93 lru_type: 1 00010000:00010000:1.0:1775764772.592007:0:11210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000083f2c617/0x36f9bdc9e41caaf4 lrc: 1/0,0 mode: --/PW res: [0x240000403:0x54:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee756 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 69 lru_type: 1 00010000:00010000:1.0:1775764772.592014:0:11210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000083f2c617/0x36f9bdc9e41caaf4 lrc: 0/0,0 mode: --/PW res: [0x240000403:0x54:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee756 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 69 lru_type: 1 00010000:00010000:2.0:1775764772.592017:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.592018:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.592020:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764772.592025:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764772.592026:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764772.592028:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764772.592034:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.592963:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.592970:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764772.592994:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966982528:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.593078:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966983040:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.593130:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966981504:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.593150:0:2382:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:2.0:1775764772.593155:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966982912:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.593177:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efde300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966983296:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764772.594723:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966982016:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764772.594838:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966982272:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764772.594863:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966982656:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764772.594882:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966983168:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764772.594916:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 4 async RPCs [1->0] 00000100:00100000:1.0:1775764772.594923:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966982144:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764772.594945:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966982400:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764772.594962:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966982784:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764772.594978:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966983424:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1775764772.597768:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764772.625128:0:5563:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978366464 00000100:00100000:3.0:1775764772.625144:0:5563:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978366464 00000100:00100000:0.0:1775764772.626227:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.626262:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:2.0:1775764772.626463:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966981888:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.627755:0:11142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 1105 00000100:00100000:2.0:1775764772.627767:0:11142:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0cb800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:12187:x1862022978366464:12345-192.168.202.146@tcp:104: 00010000:00010000:2.0:1775764772.627789:0:11142:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000016916f46/0x36f9bdc9e41ca624 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x45:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4edade expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 1 00010000:00010000:2.0:1775764772.627861:0:11142:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1775764772.627868:0:11142:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0cb800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:12187:x1862022978366464:12345-192.168.202.146@tcp:104: Request processed in 104us (41995us total) trans 0 rc 0/0 00000100:00100000:2.0:1775764772.627875:0:11142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 1105 00010000:00010000:2.0:1775764772.627896:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764772.627914:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000016916f46/0x36f9bdc9e41ca624 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x45:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4edade expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 1 00010000:00010000:2.0:1775764772.627923:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970fd7400 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:2.0:1775764772.627928:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000016916f46/0x36f9bdc9e41ca624 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x45:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4edade expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 1 00010000:00010000:2.0:1775764772.628016:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000016916f46/0x36f9bdc9e41ca624 lrc: 2/0,0 mode: --/PW res: [0x240000403:0x45:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4edade expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 1 00010000:00010000:2.0:1775764772.628024:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764772.628044:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000016916f46/0x36f9bdc9e41ca624 lrc: 1/0,0 mode: --/PW res: [0x240000403:0x45:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4edade expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 1 00010000:00010000:2.0:1775764772.628052:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000016916f46/0x36f9bdc9e41ca624 lrc: 0/0,0 mode: --/PW res: [0x240000403:0x45:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4edade expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 1 00010000:00010000:2.0:1775764772.628075:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.628076:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.628078:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764772.628085:0:2381:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1775764772.628098:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966983552:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764772.631222:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:2.0:1775764772.631401:0:5564:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978368512 00000100:00100000:2.0:1775764772.631412:0:5564:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978368512 00000100:00100000:2.0:1775764772.631463:0:5564:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 1106 00000100:00100000:2.0:1775764772.631473:0:5564:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0ca680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:12051:x1862022978368512:12345-192.168.202.146@tcp:104: 00010000:00010000:2.0:1775764772.631508:0:5564:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000072431b3a/0x36f9bdc9e421106c lrc: 2/0,0 mode: PR/PR res: [0x280000403:0x4b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5d438a expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 1 00010000:00010000:2.0:1775764772.631585:0:5564:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1775764772.631592:0:5564:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0ca680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:12051:x1862022978368512:12345-192.168.202.146@tcp:104: Request processed in 121us (381us total) trans 0 rc 0/0 00000100:00100000:2.0:1775764772.631600:0:5564:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 1106 00000100:00100000:0.0:1775764772.632171:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.632203:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.632220:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.632236:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.632257:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.632288:0:11146:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1775764772.632306:0:11146:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978368000 00000100:00100000:0.0:1775764772.632311:0:11146:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978368000 00000100:00100000:2.0:1775764772.632484:0:5564:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:2.0:1775764772.632510:0:5564:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978369152 00000100:00100000:2.0:1775764772.632513:0:5564:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978369152 00000100:00100000:2.0:1775764772.632534:0:5564:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978369280 00000100:00100000:2.0:1775764772.632537:0:5564:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978369280 00000100:00100000:2.0:1775764772.632547:0:5564:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978369536 00000100:00100000:2.0:1775764772.632548:0:5564:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978369536 00000100:00100000:2.0:1775764772.632557:0:5564:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978369664 00000100:00100000:2.0:1775764772.632559:0:5564:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978369664 00000100:00100000:2.0:1775764772.632566:0:5564:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978369792 00000100:00100000:2.0:1775764772.632568:0:5564:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978369792 00000100:00100000:2.0:1775764772.632579:0:5564:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 1107 00000100:00100000:2.0:1775764772.632587:0:5564:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0cad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+7:12182:x1862022978369152:12345-192.168.202.146@tcp:104: 00010000:00010000:2.0:1775764772.632599:0:5564:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000085d887c7/0x36f9bdc9e41ca314 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x3d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ed28e expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 129 lru_type: 1 00010000:00010000:2.0:1775764772.632662:0:5564:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:2.0:1775764772.632668:0:5564:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0cad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+7:12182:x1862022978369152:12345-192.168.202.146@tcp:104: Request processed in 82us (500us total) trans 0 rc 0/0 00000100:00100000:2.0:1775764772.632674:0:5564:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 1107 00000100:00100000:2.0:1775764772.632688:0:5564:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 1108 00000100:00100000:2.0:1775764772.632694:0:5564:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c9880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:12040:x1862022978369280:12345-192.168.202.146@tcp:104: 00010000:00010000:2.0:1775764772.632703:0:5564:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000030aeec1d/0x36f9bdc9e421ad75 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x658:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f4c54 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 0 00010000:00010000:2.0:1775764772.632732:0:5564:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:2.0:1775764772.632736:0:5564:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c9880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:12040:x1862022978369280:12345-192.168.202.146@tcp:104: Request processed in 43us (534us total) trans 0 rc 0/0 00000100:00100000:2.0:1775764772.632760:0:5564:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 1108 00000100:00100000:2.0:1775764772.632772:0:5564:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 1109 00000100:00100000:2.0:1775764772.632778:0:5564:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0cbb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+6:12041:x1862022978369536:12345-192.168.202.146@tcp:104: 00010000:00010000:2.0:1775764772.632787:0:5564:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c6478f03/0x36f9bdc9e41caa8b lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x53:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ee630 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 1 00010000:00010000:2.0:1775764772.632814:0:5564:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/4 locks to priority blp list, 4 blwis in pool 00000100:00100000:2.0:1775764772.632819:0:5564:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0cbb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+6:12041:x1862022978369536:12345-192.168.202.146@tcp:104: Request processed in 41us (599us total) trans 0 rc 0/0 00000100:00100000:2.0:1775764772.632824:0:5564:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 1109 00000100:00100000:2.0:1775764772.632835:0:5564:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 1110 00000100:00100000:2.0:1775764772.632840:0:5564:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0cb100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+5:6579:x1862022978369664:12345-192.168.202.146@tcp:104: 00010000:00010000:2.0:1775764772.632847:0:5564:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000e209157c/0x36f9bdc9e41ca2b9 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x3c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ed199 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 129 lru_type: 1 00010000:00010000:2.0:1775764772.632875:0:5564:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/5 locks to priority blp list, 5 blwis in pool 00000100:00100000:2.0:1775764772.632880:0:5564:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0cb100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+5:6579:x1862022978369664:12345-192.168.202.146@tcp:104: Request processed in 40us (643us total) trans 0 rc 0/0 00000100:00100000:2.0:1775764772.632885:0:5564:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 1110 00000100:00100000:2.0:1775764772.632895:0:5564:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 1111 00000100:00100000:2.0:1775764772.632914:0:5564:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9966932c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:12039:x1862022978369792:12345-192.168.202.146@tcp:104: 00010000:00010000:2.0:1775764772.632921:0:5564:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003d3d2f4a/0x36f9bdc9e4219e56 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x64d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f1d29 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 32 lru_type: 1 00010000:00010000:2.0:1775764772.632944:0:5564:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/6 locks to priority blp list, 6 blwis in pool 00000100:00100000:2.0:1775764772.632949:0:5564:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9966932c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:12039:x1862022978369792:12345-192.168.202.146@tcp:104: Request processed in 48us (691us total) trans 0 rc 0/0 00000100:00100000:2.0:1775764772.632958:0:5564:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 1111 00010000:00010000:2.0:1775764772.632987:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 5 total in blp. (5 blwis in pool) 00010000:00010000:2.0:1775764772.632993:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000072431b3a/0x36f9bdc9e421106c lrc: 2/0,0 mode: PR/PR res: [0x280000403:0x4b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5d438a expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 1 00010000:00010000:2.0:1775764772.633002:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9966d48400 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:2.0:1775764772.633007:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000072431b3a/0x36f9bdc9e421106c lrc: 3/0,0 mode: PR/PR res: [0x280000403:0x4b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x428400010000 nid: local remote: 0x2aac6a50e5d438a expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 1 00010000:00010000:2.0:1775764772.633093:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000072431b3a/0x36f9bdc9e421106c lrc: 2/0,0 mode: --/PR res: [0x280000403:0x4b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5d438a expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 1 00010000:00010000:2.0:1775764772.633101:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764772.633122:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000072431b3a/0x36f9bdc9e421106c lrc: 1/0,0 mode: --/PR res: [0x280000403:0x4b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5d438a expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 1 00010000:00010000:2.0:1775764772.633130:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000072431b3a/0x36f9bdc9e421106c lrc: 0/0,0 mode: --/PR res: [0x280000403:0x4b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5d438a expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 1 00010000:00010000:2.0:1775764772.633153:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 4 total in blp. (4 blwis in pool) 00010000:00010000:2.0:1775764772.633156:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000085d887c7/0x36f9bdc9e41ca314 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x3d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ed28e expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 129 lru_type: 1 00010000:00010000:2.0:1775764772.633162:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9971226a00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:2.0:1775764772.633166:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000085d887c7/0x36f9bdc9e41ca314 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x3d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ed28e expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 129 lru_type: 1 00010000:00010000:2.0:1775764772.633204:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000085d887c7/0x36f9bdc9e41ca314 lrc: 2/0,0 mode: --/PW res: [0x240000403:0x3d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed28e expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 129 lru_type: 1 00010000:00010000:2.0:1775764772.633211:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764772.633217:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000085d887c7/0x36f9bdc9e41ca314 lrc: 1/0,0 mode: --/PW res: [0x240000403:0x3d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed28e expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 129 lru_type: 1 00010000:00010000:2.0:1775764772.633223:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000085d887c7/0x36f9bdc9e41ca314 lrc: 0/0,0 mode: --/PW res: [0x240000403:0x3d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed28e expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 129 lru_type: 1 00010000:00010000:2.0:1775764772.633236:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 3 total in blp. (3 blwis in pool) 00010000:00010000:2.0:1775764772.633239:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000030aeec1d/0x36f9bdc9e421ad75 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x658:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f4c54 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 0 00010000:00010000:2.0:1775764772.633245:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970925200 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:2.0:1775764772.633248:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000030aeec1d/0x36f9bdc9e421ad75 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x658:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e5f4c54 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 0 00010000:00010000:2.0:1775764772.633279:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000030aeec1d/0x36f9bdc9e421ad75 lrc: 2/0,0 mode: --/PW res: [0x280000403:0x658:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f4c54 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 0 00010000:00010000:2.0:1775764772.633286:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764772.633291:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000030aeec1d/0x36f9bdc9e421ad75 lrc: 1/0,0 mode: --/PW res: [0x280000403:0x658:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f4c54 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 0 00010000:00010000:2.0:1775764772.633298:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000030aeec1d/0x36f9bdc9e421ad75 lrc: 0/0,0 mode: --/PW res: [0x280000403:0x658:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f4c54 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 0 00010000:00010000:2.0:1775764772.633311:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:2.0:1775764772.633314:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c6478f03/0x36f9bdc9e41caa8b lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x53:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ee630 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 1 00010000:00010000:2.0:1775764772.633320:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f994f3d9800 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:2.0:1775764772.633324:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c6478f03/0x36f9bdc9e41caa8b lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x53:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ee630 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 1 00010000:00010000:2.0:1775764772.633354:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c6478f03/0x36f9bdc9e41caa8b lrc: 2/0,0 mode: --/PW res: [0x240000403:0x53:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee630 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 1 00010000:00010000:2.0:1775764772.633361:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764772.633365:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c6478f03/0x36f9bdc9e41caa8b lrc: 1/0,0 mode: --/PW res: [0x240000403:0x53:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee630 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 1 00010000:00010000:2.0:1775764772.633372:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c6478f03/0x36f9bdc9e41caa8b lrc: 0/0,0 mode: --/PW res: [0x240000403:0x53:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee630 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 1 00010000:00010000:2.0:1775764772.633386:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1775764772.633389:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000e209157c/0x36f9bdc9e41ca2b9 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x3c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ed199 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 129 lru_type: 1 00010000:00010000:2.0:1775764772.633395:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9940030800 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:2.0:1775764772.633398:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000e209157c/0x36f9bdc9e41ca2b9 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x3c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ed199 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 129 lru_type: 1 00010000:00010000:2.0:1775764772.633432:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000e209157c/0x36f9bdc9e41ca2b9 lrc: 2/0,0 mode: --/PW res: [0x240000403:0x3c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed199 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 129 lru_type: 1 00010000:00010000:2.0:1775764772.633439:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764772.633443:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000e209157c/0x36f9bdc9e41ca2b9 lrc: 1/0,0 mode: --/PW res: [0x240000403:0x3c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed199 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 129 lru_type: 1 00010000:00010000:2.0:1775764772.633451:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000e209157c/0x36f9bdc9e41ca2b9 lrc: 0/0,0 mode: --/PW res: [0x240000403:0x3c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed199 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 129 lru_type: 1 00010000:00010000:2.0:1775764772.633463:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764772.633466:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003d3d2f4a/0x36f9bdc9e4219e56 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x64d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f1d29 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 32 lru_type: 1 00010000:00010000:2.0:1775764772.633472:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9971071a00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:2.0:1775764772.633475:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003d3d2f4a/0x36f9bdc9e4219e56 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x64d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e5f1d29 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 32 lru_type: 1 00000100:00100000:0.0:1775764772.633501:0:11146:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978368256 00010000:00010000:2.0:1775764772.633505:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003d3d2f4a/0x36f9bdc9e4219e56 lrc: 2/0,0 mode: --/PW res: [0x240000403:0x64d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f1d29 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 32 lru_type: 1 00000100:00100000:0.0:1775764772.633506:0:11146:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978368256 00010000:00010000:2.0:1775764772.633511:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764772.633516:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003d3d2f4a/0x36f9bdc9e4219e56 lrc: 1/0,0 mode: --/PW res: [0x240000403:0x64d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f1d29 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 32 lru_type: 1 00010000:00010000:2.0:1775764772.633523:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003d3d2f4a/0x36f9bdc9e4219e56 lrc: 0/0,0 mode: --/PW res: [0x240000403:0x64d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f1d29 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 32 lru_type: 1 00000100:00100000:0.0:1775764772.633526:0:11146:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24138 00000100:00100000:0.0:1775764772.633532:0:11146:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c9f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+5:12186:x1862022978368000:12345-192.168.202.146@tcp:104: 00010000:00010000:2.0:1775764772.633537:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.633538:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.633540:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764772.633541:0:11146:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000e3a8a87/0x36f9bdc9e41ca6b0 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x47:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4edc2e expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 43 lru_type: 1 00010000:00010000:2.0:1775764772.633547:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764772.633564:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966983680:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.633617:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966983936:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.633641:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966984192:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.633663:0:2381:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [1->0] 00000100:00100000:2.0:1775764772.633669:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966983808:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.633685:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966984064:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.633697:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966984320:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1775764772.633716:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764772.634056:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966982912:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.634069:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efde300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966983296:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.634078:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966982528:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.634087:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966983040:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1775764772.634813:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764772.634820:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764772.634824:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764772.635501:0:11146:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764772.635515:0:11146:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c9f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+5:12186:x1862022978368000:12345-192.168.202.146@tcp:104: Request processed in 1978us (9290us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764772.635525:0:11146:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24138 00010000:00010000:2.0:1775764772.635585:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764772.635592:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000e3a8a87/0x36f9bdc9e41ca6b0 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x47:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4edc2e expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 43 lru_type: 1 00010000:00010000:2.0:1775764772.635618:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9971079000 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:2.0:1775764772.635626:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000e3a8a87/0x36f9bdc9e41ca6b0 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x47:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4edc2e expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 43 lru_type: 1 00000100:00100000:0.0:1775764772.635700:0:11146:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24139 00010000:00010000:2.0:1775764772.635709:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000e3a8a87/0x36f9bdc9e41ca6b0 lrc: 2/0,0 mode: --/PW res: [0x240000403:0x47:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4edc2e expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 43 lru_type: 1 00000100:00100000:0.0:1775764772.635715:0:11146:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970e08c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+4:12181:x1862022978368256:12345-192.168.202.146@tcp:104: 00010000:00010000:2.0:1775764772.635721:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764772.635726:0:11146:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000054b8cc9/0x36f9bdc9e41caf93 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x64:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef2af expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 0 00010000:00010000:2.0:1775764772.635763:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000e3a8a87/0x36f9bdc9e41ca6b0 lrc: 1/0,0 mode: --/PW res: [0x240000403:0x47:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4edc2e expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 43 lru_type: 1 00010000:00010000:2.0:1775764772.635771:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000e3a8a87/0x36f9bdc9e41ca6b0 lrc: 0/0,0 mode: --/PW res: [0x240000403:0x47:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4edc2e expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 43 lru_type: 1 00010000:00010000:0.0:1775764772.635781:0:11146:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764772.635786:0:11146:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970e08c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+4:12181:x1862022978368256:12345-192.168.202.146@tcp:104: Request processed in 78us (9523us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764772.635791:0:11146:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24139 00010000:00010000:2.0:1775764772.635791:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764772.635794:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000054b8cc9/0x36f9bdc9e41caf93 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x64:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef2af expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 0 00010000:00010000:2.0:1775764772.635800:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9971071400 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:2.0:1775764772.635803:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000054b8cc9/0x36f9bdc9e41caf93 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x64:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ef2af expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 0 00010000:00010000:2.0:1775764772.635833:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000054b8cc9/0x36f9bdc9e41caf93 lrc: 2/0,0 mode: --/PW res: [0x240000403:0x64:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef2af expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 0 00010000:00010000:2.0:1775764772.635840:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764772.635847:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000054b8cc9/0x36f9bdc9e41caf93 lrc: 1/0,0 mode: --/PW res: [0x240000403:0x64:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef2af expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 0 00010000:00010000:2.0:1775764772.635853:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000054b8cc9/0x36f9bdc9e41caf93 lrc: 0/0,0 mode: --/PW res: [0x240000403:0x64:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef2af expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 0 00010000:00010000:2.0:1775764772.635866:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.635868:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.635869:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.635876:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764772.635938:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966984576:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.635991:0:2382:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1775764772.635997:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966984448:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764772.639841:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.639878:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.639900:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.639922:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.640023:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978370560 00000100:00100000:0.0:1775764772.640033:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978370560 00000100:00100000:0.0:1775764772.640084:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978370688 00000100:00100000:0.0:1775764772.640087:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978370688 00000100:00100000:0.0:1775764772.640098:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978370816 00000100:00100000:0.0:1775764772.640101:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978370816 00000100:00100000:0.0:1775764772.640112:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24140 00000100:00100000:0.0:1775764772.640120:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0cb480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+6:12184:x1862022978370560:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764772.640140:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000e5cf32fa/0x36f9bdc9e41ca942 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x4f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ee2b0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 103 lru_type: 1 00010000:00010000:0.0:1775764772.640201:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764772.640208:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0cb480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+6:12184:x1862022978370560:12345-192.168.202.146@tcp:104: Request processed in 88us (379us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764772.640215:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24140 00010000:00010000:2.0:1775764772.640225:0:19800:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:0.0:1775764772.640230:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978370944 00010000:00010000:2.0:1775764772.640232:0:19800:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000e5cf32fa/0x36f9bdc9e41ca942 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x4f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ee2b0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 103 lru_type: 1 00000100:00100000:0.0:1775764772.640233:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978370944 00010000:00010000:2.0:1775764772.640241:0:19800:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f994f3d8c00 already unused, calling callback (ffffffffc12102c0) 00000100:00100000:0.0:1775764772.640244:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24141 00010000:00010000:2.0:1775764772.640246:0:19800:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000e5cf32fa/0x36f9bdc9e41ca942 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x4f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ee2b0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 103 lru_type: 1 00000100:00100000:0.0:1775764772.640249:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c8a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+6:12190:x1862022978370688:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764772.640256:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000003718bf2e/0x36f9bdc9e41ca712 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x48:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4edd07 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 30 lru_type: 1 00010000:00010000:0.0:1775764772.640287:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764772.640292:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c8a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+6:12190:x1862022978370688:12345-192.168.202.146@tcp:104: Request processed in 43us (414us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764772.640297:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24141 00000100:00100000:0.0:1775764772.640307:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24142 00000100:00100000:0.0:1775764772.640312:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c8e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:12193:x1862022978370816:12345-192.168.202.146@tcp:104: 00010000:00010000:2.0:1775764772.640327:0:19800:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000e5cf32fa/0x36f9bdc9e41ca942 lrc: 2/0,0 mode: --/PW res: [0x280000403:0x4f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee2b0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 103 lru_type: 1 00010000:00010000:0.0:1775764772.640329:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000004fc99e26/0x36f9bdc9e41ca988 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x50:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ee358 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 91 lru_type: 1 00010000:00010000:2.0:1775764772.640335:0:19800:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764772.640351:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:0.0:1775764772.640356:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c8e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:12193:x1862022978370816:12345-192.168.202.146@tcp:104: Request processed in 44us (456us total) trans 0 rc 0/0 00010000:00010000:2.0:1775764772.640356:0:19800:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000e5cf32fa/0x36f9bdc9e41ca942 lrc: 1/0,0 mode: --/PW res: [0x280000403:0x4f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee2b0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 103 lru_type: 1 00000100:00100000:0.0:1775764772.640362:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24142 00010000:00010000:2.0:1775764772.640364:0:19800:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000e5cf32fa/0x36f9bdc9e41ca942 lrc: 0/0,0 mode: --/PW res: [0x280000403:0x4f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee2b0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 103 lru_type: 1 00010000:00010000:2.0:1775764772.640385:0:19800:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1775764772.640389:0:19800:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000003718bf2e/0x36f9bdc9e41ca712 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x48:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4edd07 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 30 lru_type: 1 00010000:00010000:2.0:1775764772.640395:0:19800:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9971079e00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:2.0:1775764772.640398:0:19800:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000003718bf2e/0x36f9bdc9e41ca712 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x48:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4edd07 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 30 lru_type: 1 00010000:00010000:2.0:1775764772.640437:0:19800:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000003718bf2e/0x36f9bdc9e41ca712 lrc: 2/0,0 mode: --/PW res: [0x280000403:0x48:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4edd07 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 30 lru_type: 1 00010000:00010000:2.0:1775764772.640444:0:19800:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764772.640450:0:19800:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000003718bf2e/0x36f9bdc9e41ca712 lrc: 1/0,0 mode: --/PW res: [0x280000403:0x48:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4edd07 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 30 lru_type: 1 00010000:00010000:2.0:1775764772.640457:0:19800:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000003718bf2e/0x36f9bdc9e41ca712 lrc: 0/0,0 mode: --/PW res: [0x280000403:0x48:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4edd07 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 30 lru_type: 1 00010000:00010000:2.0:1775764772.640471:0:19800:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764772.640474:0:19800:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000004fc99e26/0x36f9bdc9e41ca988 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x50:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ee358 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 91 lru_type: 1 00010000:00010000:2.0:1775764772.640481:0:19800:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970641000 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:2.0:1775764772.640484:0:19800:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000004fc99e26/0x36f9bdc9e41ca988 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x50:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ee358 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 91 lru_type: 1 00010000:00010000:2.0:1775764772.640532:0:19800:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000004fc99e26/0x36f9bdc9e41ca988 lrc: 2/0,0 mode: --/PW res: [0x280000403:0x50:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee358 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 91 lru_type: 1 00010000:00010000:2.0:1775764772.640539:0:19800:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764772.640543:0:19800:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000004fc99e26/0x36f9bdc9e41ca988 lrc: 1/0,0 mode: --/PW res: [0x280000403:0x50:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee358 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 91 lru_type: 1 00010000:00010000:2.0:1775764772.640550:0:19800:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000004fc99e26/0x36f9bdc9e41ca988 lrc: 0/0,0 mode: --/PW res: [0x280000403:0x50:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee358 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 91 lru_type: 1 00010000:00010000:2.0:1775764772.640563:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.640564:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.640566:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764772.640810:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764772.640818:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764772.640835:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966984832:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.640887:0:2382:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:3.0:1775764772.640893:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966984704:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.640920:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966984960:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764772.641628:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24143 00000100:00100000:0.0:1775764772.641642:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c9c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:16287:x1862022978370944:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764772.641655:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000000b074c4f/0x36f9bdc9e41ca862 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x4d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ee041 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 43 lru_type: 1 00010000:00010000:0.0:1775764772.641695:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764772.641700:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c9c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:16287:x1862022978370944:12345-192.168.202.146@tcp:104: Request processed in 65us (1777us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764772.641707:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24143 00000100:00100000:0.0:1775764772.641890:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966982144:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764772.641905:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966982400:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764772.641914:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966982784:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764772.641923:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966983424:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764772.641933:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966982016:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764772.641945:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966982272:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764772.641954:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966982656:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764772.641963:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966983168:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1775764772.645507:0:5572:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1775764772.645523:0:5572:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000000b074c4f/0x36f9bdc9e41ca862 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x4d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ee041 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 43 lru_type: 1 00010000:00010000:3.0:1775764772.645539:0:5572:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f996695ce00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:3.0:1775764772.645547:0:5572:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000000b074c4f/0x36f9bdc9e41ca862 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x4d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ee041 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 43 lru_type: 1 00010000:00010000:3.0:1775764772.645666:0:5572:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000000b074c4f/0x36f9bdc9e41ca862 lrc: 2/0,0 mode: --/PW res: [0x280000403:0x4d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee041 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 43 lru_type: 1 00010000:00010000:3.0:1775764772.645675:0:5572:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764772.645715:0:5572:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000000b074c4f/0x36f9bdc9e41ca862 lrc: 1/0,0 mode: --/PW res: [0x280000403:0x4d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee041 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 43 lru_type: 1 00010000:00010000:3.0:1775764772.645724:0:5572:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000000b074c4f/0x36f9bdc9e41ca862 lrc: 0/0,0 mode: --/PW res: [0x280000403:0x4d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee041 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 43 lru_type: 1 00010000:00010000:3.0:1775764772.645766:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764772.645769:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764772.645771:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764772.645788:0:2381:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1775764772.645811:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966985088:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764772.652333:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.653711:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978371328 00000100:00100000:0.0:1775764772.653726:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978371328 00000100:00100000:0.0:1775764772.653838:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24144 00000100:00100000:0.0:1775764772.653848:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e6a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:12196:x1862022978371328:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764772.653868:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000fd4caace/0x36f9bdc9e41ca8ee lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x4e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ee1d0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 52 lru_type: 1 00010000:00010000:0.0:1775764772.653968:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764772.653976:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e6a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:12196:x1862022978371328:12345-192.168.202.146@tcp:104: Request processed in 129us (1665us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764772.653983:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24144 00010000:00010000:2.0:1775764772.654019:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764772.654026:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000fd4caace/0x36f9bdc9e41ca8ee lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x4e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ee1d0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 52 lru_type: 1 00010000:00010000:2.0:1775764772.654034:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f994f3d8200 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:2.0:1775764772.654039:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000fd4caace/0x36f9bdc9e41ca8ee lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x4e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ee1d0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 52 lru_type: 1 00010000:00010000:2.0:1775764772.654129:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000fd4caace/0x36f9bdc9e41ca8ee lrc: 2/0,0 mode: --/PW res: [0x280000403:0x4e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee1d0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 52 lru_type: 1 00010000:00010000:2.0:1775764772.654137:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764772.654163:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000fd4caace/0x36f9bdc9e41ca8ee lrc: 1/0,0 mode: --/PW res: [0x280000403:0x4e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee1d0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 52 lru_type: 1 00010000:00010000:2.0:1775764772.654172:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000fd4caace/0x36f9bdc9e41ca8ee lrc: 0/0,0 mode: --/PW res: [0x280000403:0x4e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee1d0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 52 lru_type: 1 00000100:00100000:3.0:1775764772.654188:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efde300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966985216:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1775764772.654194:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.654196:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.654197:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764772.661526:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966984704:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.661544:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966984960:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.661553:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966984832:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.661563:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966984448:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.661570:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966984576:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.661790:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966983808:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.661801:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966984064:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.661808:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966984320:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.661819:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966983680:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.661827:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966983936:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.661834:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966984192:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.661841:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966983552:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764772.662825:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.662951:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764772.663117:0:11131:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1775764772.663138:0:11131:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978371968 00000100:00100000:1.0:1775764772.663147:0:11131:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978371968 00000100:00100000:1.0:1775764772.663194:0:11131:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978372096 00000100:00100000:1.0:1775764772.663198:0:11131:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978372096 00000100:00100000:1.0:1775764772.663222:0:11131:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24145 00000100:00100000:1.0:1775764772.663230:0:11131:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0caa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:16465:x1862022978371968:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764772.663248:0:11131:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000004f97dc86/0x36f9bdc9e41ca30d lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x3c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ed287 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 109 lru_type: 1 00010000:00010000:1.0:1775764772.663337:0:11131:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764772.663343:0:11131:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0caa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:16465:x1862022978371968:12345-192.168.202.146@tcp:104: Request processed in 114us (526us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764772.663351:0:11131:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24145 00000100:00100000:1.0:1775764772.663367:0:11131:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24146 00000100:00100000:1.0:1775764772.663373:0:11131:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9966bf5050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:6579:x1862022978372096:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764772.663382:0:11131:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a2c7fa1e/0x36f9bdc9e42134e6 lrc: 2/0,0 mode: PR/PR res: [0x240000403:0x3a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5dc3eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00010000:00010000:1.0:1775764772.663417:0:11131:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1775764772.663422:0:11131:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9966bf5050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:6579:x1862022978372096:12345-192.168.202.146@tcp:104: Request processed in 50us (472us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764772.663428:0:11131:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24146 00010000:00010000:2.0:1775764772.663513:0:11138:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1775764772.663519:0:11138:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000004f97dc86/0x36f9bdc9e41ca30d lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x3c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ed287 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 109 lru_type: 1 00010000:00010000:2.0:1775764772.663527:0:11138:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9971078e00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:2.0:1775764772.663533:0:11138:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000004f97dc86/0x36f9bdc9e41ca30d lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x3c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ed287 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 109 lru_type: 1 00010000:00010000:2.0:1775764772.663629:0:11138:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000004f97dc86/0x36f9bdc9e41ca30d lrc: 2/0,0 mode: --/PW res: [0x280000403:0x3c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed287 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 109 lru_type: 1 00010000:00010000:2.0:1775764772.663637:0:11138:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764772.663660:0:11138:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000004f97dc86/0x36f9bdc9e41ca30d lrc: 1/0,0 mode: --/PW res: [0x280000403:0x3c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed287 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 109 lru_type: 1 00010000:00010000:2.0:1775764772.663669:0:11138:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000004f97dc86/0x36f9bdc9e41ca30d lrc: 0/0,0 mode: --/PW res: [0x280000403:0x3c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed287 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 109 lru_type: 1 00010000:00010000:2.0:1775764772.663689:0:11138:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764772.663692:0:11138:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a2c7fa1e/0x36f9bdc9e42134e6 lrc: 2/0,0 mode: PR/PR res: [0x240000403:0x3a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5dc3eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00010000:00010000:2.0:1775764772.663704:0:11138:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99710c9200 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:2.0:1775764772.663708:0:11138:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a2c7fa1e/0x36f9bdc9e42134e6 lrc: 3/0,0 mode: PR/PR res: [0x240000403:0x3a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x428400010000 nid: local remote: 0x2aac6a50e5dc3eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00010000:00010000:2.0:1775764772.663779:0:11138:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a2c7fa1e/0x36f9bdc9e42134e6 lrc: 2/0,0 mode: --/PR res: [0x240000403:0x3a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5dc3eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00010000:00010000:2.0:1775764772.663788:0:11138:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764772.663794:0:11138:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a2c7fa1e/0x36f9bdc9e42134e6 lrc: 1/0,0 mode: --/PR res: [0x240000403:0x3a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5dc3eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00010000:00010000:2.0:1775764772.663800:0:11138:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a2c7fa1e/0x36f9bdc9e42134e6 lrc: 0/0,0 mode: --/PR res: [0x240000403:0x3a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5dc3eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00010000:00010000:2.0:1775764772.663815:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.663816:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.663818:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764772.663834:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966985472:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.663889:0:2381:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1775764772.663895:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966985344:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1775764772.664303:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764772.670897:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.671685:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.689107:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764772.691691:0:11131:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978372352 00000100:00100000:1.0:1775764772.691710:0:11131:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978372352 00000100:00100000:1.0:1775764772.691803:0:11131:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978372608 00000100:00100000:1.0:1775764772.691806:0:11131:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978372608 00000100:00100000:1.0:1775764772.691838:0:11131:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24147 00000100:00100000:1.0:1775764772.691848:0:11131:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c9f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:12039:x1862022978372352:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764772.691866:0:11131:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000719d22c5/0x36f9bdc9e41caa30 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x51:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ee550 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 22 lru_type: 1 00010000:00010000:1.0:1775764772.691952:0:11131:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764772.691958:0:11131:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c9f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:12039:x1862022978372352:12345-192.168.202.146@tcp:104: Request processed in 112us (21071us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764772.691965:0:11131:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24147 00010000:00010000:3.0:1775764772.691975:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1775764772.691981:0:11131:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24148 00010000:00010000:3.0:1775764772.691982:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000719d22c5/0x36f9bdc9e41caa30 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x51:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ee550 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 22 lru_type: 1 00000100:00100000:1.0:1775764772.691987:0:11131:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0cb480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:12040:x1862022978372608:12345-192.168.202.146@tcp:104: 00010000:00010000:3.0:1775764772.691991:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99700e4400 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:1.0:1775764772.691996:0:11131:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000aac2e8d4/0x36f9bdc9e41ca193 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x39:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ece7b expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00010000:00010000:3.0:1775764772.691997:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000719d22c5/0x36f9bdc9e41caa30 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x51:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ee550 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 22 lru_type: 1 00010000:00010000:1.0:1775764772.692028:0:11131:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764772.692033:0:11131:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0cb480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:12040:x1862022978372608:12345-192.168.202.146@tcp:104: Request processed in 47us (20354us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764772.692041:0:11131:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24148 00010000:00010000:3.0:1775764772.692079:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000719d22c5/0x36f9bdc9e41caa30 lrc: 2/0,0 mode: --/PW res: [0x280000403:0x51:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee550 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 22 lru_type: 1 00010000:00010000:3.0:1775764772.692087:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764772.692105:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000719d22c5/0x36f9bdc9e41caa30 lrc: 1/0,0 mode: --/PW res: [0x280000403:0x51:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee550 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 22 lru_type: 1 00010000:00010000:3.0:1775764772.692113:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000719d22c5/0x36f9bdc9e41caa30 lrc: 0/0,0 mode: --/PW res: [0x280000403:0x51:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee550 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 22 lru_type: 1 00010000:00010000:3.0:1775764772.692135:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1775764772.692138:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000aac2e8d4/0x36f9bdc9e41ca193 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x39:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ece7b expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00010000:00010000:3.0:1775764772.692146:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9971070000 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:3.0:1775764772.692149:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000aac2e8d4/0x36f9bdc9e41ca193 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x39:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ece7b expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00010000:00010000:2.0:1775764772.692168:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764772.692181:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000aac2e8d4/0x36f9bdc9e41ca193 lrc: 2/0,0 mode: --/PW res: [0x240000403:0x39:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ece7b expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00000100:00100000:2.0:1775764772.692182:0:2381:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00010000:00010000:3.0:1775764772.692187:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764772.692192:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000aac2e8d4/0x36f9bdc9e41ca193 lrc: 1/0,0 mode: --/PW res: [0x240000403:0x39:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ece7b expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00000100:00100000:2.0:1775764772.692196:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966985600:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1775764772.692199:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000aac2e8d4/0x36f9bdc9e41ca193 lrc: 0/0,0 mode: --/PW res: [0x240000403:0x39:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ece7b expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00010000:00010000:3.0:1775764772.692210:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764772.692211:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764772.692213:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764772.692256:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966985728:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764772.696480:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978371456 00000100:00100000:0.0:1775764772.696497:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978371456 00000100:00100000:0.0:1775764772.696561:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24149 00000100:00100000:0.0:1775764772.696571:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c8a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:12180:x1862022978371456:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764772.696590:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000084562520/0x36f9bdc9e41ca353 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x3d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ed32f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 108 lru_type: 1 00010000:00010000:0.0:1775764772.696727:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764772.696735:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c8a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:12180:x1862022978371456:12345-192.168.202.146@tcp:104: Request processed in 166us (7638us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764772.696777:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24149 00000100:00100000:0.0:1775764772.696869:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.696887:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.696905:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.696938:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.696953:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.696971:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.696988:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.697004:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.697041:0:5562:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:2.0:1775764772.697244:0:11142:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978374144 00000100:00100000:2.0:1775764772.697249:0:11142:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978374144 00000100:00100000:2.0:1775764772.697270:0:11142:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978374272 00000100:00100000:2.0:1775764772.697273:0:11142:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978374272 00000100:00100000:2.0:1775764772.697292:0:11142:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978374400 00000100:00100000:2.0:1775764772.697295:0:11142:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978374400 00000100:00100000:2.0:1775764772.697305:0:11142:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978374528 00000100:00100000:2.0:1775764772.697307:0:11142:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978374528 00000100:00100000:2.0:1775764772.697326:0:11142:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 1112 00000100:00100000:2.0:1775764772.697334:0:11142:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0cb800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+5:12181:x1862022978374144:12345-192.168.202.146@tcp:104: 00010000:00010000:2.0:1775764772.697346:0:11142:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ba4d6611/0x36f9bdc9e4219a51 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x649:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f106b expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 0 00000100:00100000:0.0:1775764772.697495:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978372864 00000100:00100000:0.0:1775764772.697499:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978372864 00000100:00100000:0.0:1775764772.697516:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978371584 00000100:00100000:0.0:1775764772.697518:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978371584 00000100:00100000:0.0:1775764772.697526:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978373888 00000100:00100000:0.0:1775764772.697528:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978373888 00000100:00100000:0.0:1775764772.697535:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978374016 00000100:00100000:0.0:1775764772.697537:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978374016 00000100:00100000:0.0:1775764772.697548:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24150 00000100:00100000:0.0:1775764772.697554:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c8e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+6:12182:x1862022978372864:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764772.697562:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000d1d1380c/0x36f9bdc9e41caadf lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x53:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ee748 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 28 lru_type: 1 00000100:00100000:1.0:1775764772.698447:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24151 00000100:00100000:1.0:1775764772.698462:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c9c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+8:6580:x1862022978371584:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764772.698473:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d3da0fa6/0x36f9bdc9e4213708 lrc: 2/0,0 mode: PR/PR res: [0x240000403:0x3b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5dcb1c expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:3.0:1775764772.706156:0:5572:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 3 total in blp. (3 blwis in pool) 00010000:00010000:3.0:1775764772.706171:0:5572:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000084562520/0x36f9bdc9e41ca353 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x3d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ed32f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 108 lru_type: 1 00010000:00010000:3.0:1775764772.706184:0:5572:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9971071800 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:3.0:1775764772.706194:0:5572:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000084562520/0x36f9bdc9e41ca353 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x3d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ed32f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 108 lru_type: 1 00010000:00010000:3.0:1775764772.706321:0:5572:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000084562520/0x36f9bdc9e41ca353 lrc: 2/0,0 mode: --/PW res: [0x280000403:0x3d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed32f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 108 lru_type: 1 00010000:00010000:3.0:1775764772.706330:0:5572:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764772.706367:0:5572:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000084562520/0x36f9bdc9e41ca353 lrc: 1/0,0 mode: --/PW res: [0x280000403:0x3d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed32f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 108 lru_type: 1 00010000:00010000:3.0:1775764772.706376:0:5572:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000084562520/0x36f9bdc9e41ca353 lrc: 0/0,0 mode: --/PW res: [0x280000403:0x3d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed32f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 108 lru_type: 1 00010000:00010000:3.0:1775764772.706399:0:5572:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:3.0:1775764772.706403:0:5572:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ba4d6611/0x36f9bdc9e4219a51 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x649:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f106b expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 0 00010000:00010000:3.0:1775764772.706410:0:5572:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f994ed8d600 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:3.0:1775764772.706414:0:5572:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ba4d6611/0x36f9bdc9e4219a51 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x649:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e5f106b expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 0 00010000:00010000:3.0:1775764772.706451:0:5572:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ba4d6611/0x36f9bdc9e4219a51 lrc: 2/0,0 mode: --/PW res: [0x240000403:0x649:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f106b expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 0 00010000:00010000:3.0:1775764772.706462:0:5572:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764772.706469:0:5572:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ba4d6611/0x36f9bdc9e4219a51 lrc: 1/0,0 mode: --/PW res: [0x240000403:0x649:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f106b expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 0 00010000:00010000:3.0:1775764772.706476:0:5572:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ba4d6611/0x36f9bdc9e4219a51 lrc: 0/0,0 mode: --/PW res: [0x240000403:0x649:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f106b expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 0 00010000:00010000:3.0:1775764772.706492:0:5572:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1775764772.706497:0:5572:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000d1d1380c/0x36f9bdc9e41caadf lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x53:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ee748 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 28 lru_type: 1 00010000:00010000:3.0:1775764772.706503:0:5572:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970640400 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:3.0:1775764772.706506:0:5572:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000d1d1380c/0x36f9bdc9e41caadf lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x53:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ee748 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 28 lru_type: 1 00010000:00010000:3.0:1775764772.706614:0:5572:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000d1d1380c/0x36f9bdc9e41caadf lrc: 2/0,0 mode: --/PW res: [0x280000403:0x53:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee748 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 28 lru_type: 1 00010000:00010000:3.0:1775764772.706622:0:5572:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764772.706627:0:5572:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000d1d1380c/0x36f9bdc9e41caadf lrc: 1/0,0 mode: --/PW res: [0x280000403:0x53:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee748 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 28 lru_type: 1 00010000:00010000:3.0:1775764772.706633:0:5572:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000d1d1380c/0x36f9bdc9e41caadf lrc: 0/0,0 mode: --/PW res: [0x280000403:0x53:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee748 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 28 lru_type: 1 00010000:00010000:3.0:1775764772.706648:0:5572:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1775764772.706652:0:5572:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d3da0fa6/0x36f9bdc9e4213708 lrc: 2/0,0 mode: PR/PR res: [0x240000403:0x3b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5dcb1c expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:3.0:1775764772.706659:0:5572:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99710c9e00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:3.0:1775764772.706662:0:5572:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d3da0fa6/0x36f9bdc9e4213708 lrc: 3/0,0 mode: PR/PR res: [0x240000403:0x3b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x428400010000 nid: local remote: 0x2aac6a50e5dcb1c expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:3.0:1775764772.706697:0:5572:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d3da0fa6/0x36f9bdc9e4213708 lrc: 2/0,0 mode: --/PR res: [0x240000403:0x3b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5dcb1c expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:3.0:1775764772.706704:0:5572:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764772.706709:0:5572:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d3da0fa6/0x36f9bdc9e4213708 lrc: 1/0,0 mode: --/PR res: [0x240000403:0x3b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5dcb1c expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:3.0:1775764772.706715:0:5572:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d3da0fa6/0x36f9bdc9e4213708 lrc: 0/0,0 mode: --/PR res: [0x240000403:0x3b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5dcb1c expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:3.0:1775764772.706729:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764772.706731:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764772.706733:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.708124:0:11142:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/0 locks to priority blp list, 0 blwis in pool 00000100:00100000:2.0:1775764772.708143:0:11142:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0cb800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+8:12181:x1862022978374144:12345-192.168.202.146@tcp:104: Request processed in 10803us (11181us total) trans 0 rc 0/0 00000100:00100000:2.0:1775764772.708154:0:11142:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 1112 00000100:00100000:2.0:1775764772.708202:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966985856:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1775764772.708538:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764772.708574:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966986112:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.708618:0:2382:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:2.0:1775764772.708627:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966985984:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.708651:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966986240:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.708707:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966985344:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.708720:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966985472:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.708732:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efde300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966985216:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.708772:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966985088:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.708805:0:5564:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 1113 00000100:00100000:2.0:1775764772.708816:0:5564:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0ca680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+7:12051:x1862022978374272:12345-192.168.202.146@tcp:104: 00010000:00010000:2.0:1775764772.708833:0:5564:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a315e706/0x36f9bdc9e42196fb lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x646:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f05eb expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 0 00010000:00010000:2.0:1775764772.708873:0:5564:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1775764772.708877:0:5564:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0ca680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+7:12051:x1862022978374272:12345-192.168.202.146@tcp:104: Request processed in 66us (11906us total) trans 0 rc 0/0 00000100:00100000:2.0:1775764772.708884:0:5564:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 1113 00000100:00100000:2.0:1775764772.708897:0:5564:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 1114 00000100:00100000:2.0:1775764772.708903:0:5564:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0cad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+6:12188:x1862022978374400:12345-192.168.202.146@tcp:104: 00010000:00010000:2.0:1775764772.708912:0:5564:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000024570130/0x36f9bdc9e41ca265 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x3b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ed08f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00010000:00010000:2.0:1775764772.708938:0:5564:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:2.0:1775764772.708943:0:5564:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0cad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+6:12188:x1862022978374400:12345-192.168.202.146@tcp:104: Request processed in 41us (11954us total) trans 0 rc 0/0 00000100:00100000:2.0:1775764772.708948:0:5564:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 1114 00000100:00100000:2.0:1775764772.708959:0:5564:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 1115 00000100:00100000:2.0:1775764772.708966:0:5564:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c9880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+5:12215:x1862022978374528:12345-192.168.202.146@tcp:104: 00010000:00010000:2.0:1775764772.708974:0:5564:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000bb0f9183/0x36f9bdc9e41ca21f lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x3a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ecffc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 21 lru_type: 1 00010000:00010000:2.0:1775764772.709004:0:5564:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:2.0:1775764772.709009:0:5564:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c9880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+5:12215:x1862022978374528:12345-192.168.202.146@tcp:104: Request processed in 44us (12004us total) trans 0 rc 0/0 00000100:00100000:2.0:1775764772.709014:0:5564:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 1115 00010000:00010000:2.0:1775764772.709027:0:11210:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:2.0:1775764772.709032:0:11210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a315e706/0x36f9bdc9e42196fb lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x646:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f05eb expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 0 00010000:00010000:2.0:1775764772.709041:0:11210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99706ea600 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:2.0:1775764772.709057:0:11210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a315e706/0x36f9bdc9e42196fb lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x646:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e5f05eb expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 0 00010000:00010000:2.0:1775764772.709123:0:11210:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a315e706/0x36f9bdc9e42196fb lrc: 2/0,0 mode: --/PW res: [0x240000403:0x646:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f05eb expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 0 00010000:00010000:2.0:1775764772.709130:0:11210:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764772.709143:0:11210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a315e706/0x36f9bdc9e42196fb lrc: 1/0,0 mode: --/PW res: [0x240000403:0x646:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f05eb expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 0 00010000:00010000:2.0:1775764772.709151:0:11210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a315e706/0x36f9bdc9e42196fb lrc: 0/0,0 mode: --/PW res: [0x240000403:0x646:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f05eb expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 0 00010000:00010000:2.0:1775764772.709170:0:11210:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1775764772.709173:0:11210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000024570130/0x36f9bdc9e41ca265 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x3b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ed08f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00010000:00010000:2.0:1775764772.709185:0:11210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9971079400 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:2.0:1775764772.709188:0:11210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000024570130/0x36f9bdc9e41ca265 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x3b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ed08f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00010000:00010000:2.0:1775764772.709213:0:11210:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000024570130/0x36f9bdc9e41ca265 lrc: 2/0,0 mode: --/PW res: [0x280000403:0x3b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed08f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00010000:00010000:2.0:1775764772.709219:0:11210:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764772.709223:0:11210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000024570130/0x36f9bdc9e41ca265 lrc: 1/0,0 mode: --/PW res: [0x280000403:0x3b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed08f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00010000:00010000:2.0:1775764772.709231:0:11210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000024570130/0x36f9bdc9e41ca265 lrc: 0/0,0 mode: --/PW res: [0x280000403:0x3b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed08f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00010000:00010000:2.0:1775764772.709243:0:11210:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764772.709245:0:11210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000bb0f9183/0x36f9bdc9e41ca21f lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x3a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ecffc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 21 lru_type: 1 00010000:00010000:2.0:1775764772.709251:0:11210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f994f3b9e00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:2.0:1775764772.709255:0:11210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000bb0f9183/0x36f9bdc9e41ca21f lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x3a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ecffc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 21 lru_type: 1 00010000:00010000:2.0:1775764772.709278:0:11210:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000bb0f9183/0x36f9bdc9e41ca21f lrc: 2/0,0 mode: --/PW res: [0x280000403:0x3a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ecffc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 21 lru_type: 1 00010000:00010000:2.0:1775764772.709299:0:11210:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764772.709304:0:11210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000bb0f9183/0x36f9bdc9e41ca21f lrc: 1/0,0 mode: --/PW res: [0x280000403:0x3a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ecffc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 21 lru_type: 1 00010000:00010000:2.0:1775764772.709311:0:11210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000bb0f9183/0x36f9bdc9e41ca21f lrc: 0/0,0 mode: --/PW res: [0x280000403:0x3a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ecffc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 21 lru_type: 1 00010000:00010000:3.0:1775764772.709594:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.709687:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.709689:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.709690:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764772.709705:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efde300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966986368:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.709831:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdf800 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966981632:192.168.202.146@tcp:37:rm.0 00010000:00010000:3.0:1775764772.710461:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eac00/0x36f9bdc9e421c6d5 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5faa17 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764772.710475:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eac00/0x36f9bdc9e421c6d5 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faa17 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764772.710524:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2b2:0x0](ffff8f9952d8ba88) pos/size18446744073709551614/7168 32bit_api 0 00000080:00200008:3.0:1775764772.710543:0:19896:0:(file.c:447:ll_file_release()) START file PM:[0x200000401:0x2b2:0x0](ffff8f9952d8ba88), flags 704000 00010000:00010000:3.0:1775764772.710565:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970abfa00/0x36f9bdc9e41ca9ff lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2b2:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ee4c4 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 241 lru_type: 1 00000080:00200000:3.0:1775764772.710597:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764772.710919:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PM, flags=0 00000080:00200000:3.0:1775764772.710927:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 3 00000080:00200000:3.0:1775764772.710933:0:19896:0:(namei.c:2305:ll_rmdir()) VFS Op:name=PM, dir=[0x200000401:0x26c:0x0](ffff8f9952db6c08) 00010000:00010000:3.0:1775764772.710961:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eac00/0x36f9bdc9e421c6d5 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5faa17 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764772.711030:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970abf200/0x36f9bdc9e41caa45 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x11/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e4ee588 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 102 lru_type: 1 00010000:00010000:3.0:1775764772.711057:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970abfa00/0x36f9bdc9e41ca9ff lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2b2:0x0].0x0 bits 0x5/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e4ee4c4 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 241 lru_type: 1 00000002:00100000:3.0:1775764772.711099:0:19896:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f994f2e4e00 x1862022950356352/t4294969644(4294969644) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764071 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000020:00100000:3.0:1775764772.711147:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:3.0:1775764772.711152:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970eded80 x1862022966986880/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:3.0:1775764772.711171:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966986880:192.168.202.146@tcp:35:rm.0 00000100:00100000:2.0:1775764772.711543:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966986624:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.711582:0:2382:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1775764772.711588:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966986496:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1775764772.711637:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764772.711720:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413d80 going to sleep for 11 seconds 00010000:00010000:0.0:1775764772.711830:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/0 locks to priority blp list, 0 blwis in pool 00000100:00100000:0.0:1775764772.711842:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c8e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:12182:x1862022978372864:12345-192.168.202.146@tcp:104: Request processed in 14285us (14971us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764772.711852:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24150 00000100:00100000:0.0:1775764772.711882:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24152 00000100:00100000:0.0:1775764772.711893:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0caa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+6:16287:x1862022978373888:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764772.711905:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f98c8360/0x36f9bdc9e41ca495 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x41:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ed69a expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 95 lru_type: 1 00010000:00010000:0.0:1775764772.711950:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764772.711955:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0caa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+6:16287:x1862022978373888:12345-192.168.202.146@tcp:104: Request processed in 67us (15050us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764772.711961:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24152 00000100:00100000:0.0:1775764772.711973:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24153 00000100:00100000:0.0:1775764772.711980:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970e09050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:12193:x1862022978374016:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764772.711988:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000083a42736/0x36f9bdc9e4212bbd lrc: 2/0,0 mode: PR/PR res: [0x240000403:0x42:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5da6be expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00010000:00010000:0.0:1775764772.712013:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:0.0:1775764772.712018:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970e09050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:12193:x1862022978374016:12345-192.168.202.146@tcp:104: Request processed in 39us (15080us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764772.712023:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24153 00010000:00010000:0.0:1775764772.712037:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1775764772.712042:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f98c8360/0x36f9bdc9e41ca495 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x41:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ed69a expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 95 lru_type: 1 00010000:00010000:0.0:1775764772.712062:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9971078200 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:0.0:1775764772.712067:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f98c8360/0x36f9bdc9e41ca495 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x41:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ed69a expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 95 lru_type: 1 00010000:00010000:0.0:1775764772.712132:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f98c8360/0x36f9bdc9e41ca495 lrc: 2/0,0 mode: --/PW res: [0x240000403:0x41:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed69a expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 95 lru_type: 1 00010000:00010000:0.0:1775764772.712139:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764772.712158:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f98c8360/0x36f9bdc9e41ca495 lrc: 1/0,0 mode: --/PW res: [0x240000403:0x41:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed69a expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 95 lru_type: 1 00010000:00010000:0.0:1775764772.712166:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f98c8360/0x36f9bdc9e41ca495 lrc: 0/0,0 mode: --/PW res: [0x240000403:0x41:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed69a expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 95 lru_type: 1 00010000:00010000:0.0:1775764772.712183:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1775764772.712186:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000083a42736/0x36f9bdc9e4212bbd lrc: 2/0,0 mode: PR/PR res: [0x240000403:0x42:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5da6be expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00010000:00010000:0.0:1775764772.712192:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9966c08e00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:0.0:1775764772.712195:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000083a42736/0x36f9bdc9e4212bbd lrc: 3/0,0 mode: PR/PR res: [0x240000403:0x42:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x428400010000 nid: local remote: 0x2aac6a50e5da6be expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00010000:00010000:0.0:1775764772.712221:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000083a42736/0x36f9bdc9e4212bbd lrc: 2/0,0 mode: --/PR res: [0x240000403:0x42:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5da6be expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00010000:00010000:0.0:1775764772.712228:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764772.712233:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000083a42736/0x36f9bdc9e4212bbd lrc: 1/0,0 mode: --/PR res: [0x240000403:0x42:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5da6be expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00010000:00010000:0.0:1775764772.712245:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000083a42736/0x36f9bdc9e4212bbd lrc: 0/0,0 mode: --/PR res: [0x240000403:0x42:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5da6be expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00010000:00010000:0.0:1775764772.712258:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764772.712259:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764772.712261:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764772.712271:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966987008:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1775764772.712513:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764772.712559:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1775764772.712565:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966987136:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764772.712638:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.712661:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.712674:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.712686:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.712776:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.712790:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978374656 00000100:00100000:0.0:1775764772.712797:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978374656 00000100:00100000:0.0:1775764772.712817:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978374912 00000100:00100000:0.0:1775764772.712819:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978374912 00000100:00100000:0.0:1775764772.712826:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978375424 00000100:00100000:0.0:1775764772.712828:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978375424 00000100:00100000:0.0:1775764772.712835:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978375552 00000100:00100000:0.0:1775764772.712837:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978375552 00000100:00100000:0.0:1775764772.712844:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978375680 00000100:00100000:0.0:1775764772.712846:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978375680 00000100:00100000:0.0:1775764772.712855:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24154 00000100:00100000:0.0:1775764772.712862:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c8380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+6:12196:x1862022978374656:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764772.712875:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000064603db/0x36f9bdc9e41ca1cb lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x39:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ecf15 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 0 00010000:00010000:0.0:1775764772.712905:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764772.712911:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c8380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+6:12196:x1862022978374656:12345-192.168.202.146@tcp:104: Request processed in 50us (274us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764772.712916:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24154 00000100:00100000:0.0:1775764772.712929:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24155 00000100:00100000:0.0:1775764772.712934:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c9180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+6:6579:x1862022978374912:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764772.712942:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a6b29ac0/0x36f9bdc9e42125a4 lrc: 2/0,0 mode: PR/PR res: [0x240000403:0x3f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5d925f expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:0.0:1775764772.712964:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:0.0:1775764772.712968:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c9180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+6:6579:x1862022978374912:12345-192.168.202.146@tcp:104: Request processed in 35us (306us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764772.712975:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24155 00000100:00100000:0.0:1775764772.712987:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24156 00000100:00100000:0.0:1775764772.712992:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0ca300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:12187:x1862022978375424:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764772.712998:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000a6966465/0x36f9bdc9e4219aba lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x649:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 12288->16383) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f11b4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:0.0:1775764772.713020:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:0.0:1775764772.713025:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0ca300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:12187:x1862022978375424:12345-192.168.202.146@tcp:104: Request processed in 33us (351us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764772.713031:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24156 00000100:00100000:0.0:1775764772.713041:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24157 00010000:00010000:1.0:1775764772.713158:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:1.0:1775764772.713166:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c9c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:6580:x1862022978371584:12345-192.168.202.146@tcp:104: Request processed in 14710us (16277us total) trans 0 rc 0/0 00010000:00010000:3.0:1775764772.713270:0:11138:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:3.0:1775764772.713276:0:11138:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000064603db/0x36f9bdc9e41ca1cb lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x39:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ecf15 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 0 00010000:00010000:3.0:1775764772.713298:0:11138:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9971078c00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:3.0:1775764772.713302:0:11138:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000064603db/0x36f9bdc9e41ca1cb lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x39:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ecf15 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 0 00010000:00010000:3.0:1775764772.713376:0:11138:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000064603db/0x36f9bdc9e41ca1cb lrc: 2/0,0 mode: --/PW res: [0x280000403:0x39:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ecf15 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 0 00010000:00010000:3.0:1775764772.713383:0:11138:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764772.713397:0:11138:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000064603db/0x36f9bdc9e41ca1cb lrc: 1/0,0 mode: --/PW res: [0x280000403:0x39:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ecf15 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 0 00010000:00010000:3.0:1775764772.713404:0:11138:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000064603db/0x36f9bdc9e41ca1cb lrc: 0/0,0 mode: --/PW res: [0x280000403:0x39:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ecf15 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 0 00010000:00010000:3.0:1775764772.713422:0:11138:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1775764772.713425:0:11138:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a6b29ac0/0x36f9bdc9e42125a4 lrc: 2/0,0 mode: PR/PR res: [0x240000403:0x3f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5d925f expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:3.0:1775764772.713432:0:11138:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9966c09a00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:3.0:1775764772.713435:0:11138:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a6b29ac0/0x36f9bdc9e42125a4 lrc: 3/0,0 mode: PR/PR res: [0x240000403:0x3f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x428400010000 nid: local remote: 0x2aac6a50e5d925f expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:3.0:1775764772.713472:0:11138:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a6b29ac0/0x36f9bdc9e42125a4 lrc: 2/0,0 mode: --/PR res: [0x240000403:0x3f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5d925f expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:3.0:1775764772.713479:0:11138:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764772.713485:0:11138:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a6b29ac0/0x36f9bdc9e42125a4 lrc: 1/0,0 mode: --/PR res: [0x240000403:0x3f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5d925f expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:3.0:1775764772.713493:0:11138:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a6b29ac0/0x36f9bdc9e42125a4 lrc: 0/0,0 mode: --/PR res: [0x240000403:0x3f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5d925f expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:3.0:1775764772.713506:0:11138:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1775764772.713509:0:11138:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000a6966465/0x36f9bdc9e4219aba lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x649:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 12288->16383) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f11b4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:3.0:1775764772.713515:0:11138:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99700e4c00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:3.0:1775764772.713519:0:11138:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000a6966465/0x36f9bdc9e4219aba lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x649:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 12288->16383) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e5f11b4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:3.0:1775764772.713551:0:11138:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000a6966465/0x36f9bdc9e4219aba lrc: 2/0,0 mode: --/PW res: [0x280000403:0x649:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 12288->16383) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f11b4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:3.0:1775764772.713557:0:11138:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764772.713561:0:11138:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000a6966465/0x36f9bdc9e4219aba lrc: 1/0,0 mode: --/PW res: [0x280000403:0x649:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 12288->16383) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f11b4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:3.0:1775764772.713568:0:11138:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000a6966465/0x36f9bdc9e4219aba lrc: 0/0,0 mode: --/PW res: [0x280000403:0x649:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 12288->16383) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f11b4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:3.0:1775764772.713581:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764772.713583:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764772.713585:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764772.713808:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c8000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:12184:x1862022978375552:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764772.713817:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000009e8ce381/0x36f9bdc9e41ca973 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x50:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ee327 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 129 lru_type: 1 00010000:00010000:0.0:1775764772.713852:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764772.713857:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c8000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:12184:x1862022978375552:12345-192.168.202.146@tcp:104: Request processed in 56us (1171us total) trans 0 rc 0/0 00010000:00010000:3.0:1775764772.713896:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1775764772.713901:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000009e8ce381/0x36f9bdc9e41ca973 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x50:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ee327 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 129 lru_type: 1 00010000:00010000:3.0:1775764772.713909:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970640c00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:3.0:1775764772.713912:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000009e8ce381/0x36f9bdc9e41ca973 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x50:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ee327 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 129 lru_type: 1 00010000:00010000:3.0:1775764772.713951:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000009e8ce381/0x36f9bdc9e41ca973 lrc: 2/0,0 mode: --/PW res: [0x240000403:0x50:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee327 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 129 lru_type: 1 00010000:00010000:3.0:1775764772.713959:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764772.713964:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000009e8ce381/0x36f9bdc9e41ca973 lrc: 1/0,0 mode: --/PW res: [0x240000403:0x50:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee327 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 129 lru_type: 1 00010000:00010000:3.0:1775764772.713971:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000009e8ce381/0x36f9bdc9e41ca973 lrc: 0/0,0 mode: --/PW res: [0x240000403:0x50:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee327 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 129 lru_type: 1 00010000:00010000:3.0:1775764772.713986:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764772.713988:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764772.713989:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.714070:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.714075:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764772.714091:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966987392:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.714134:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966987648:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.714156:0:2382:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:2.0:1775764772.714161:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966987264:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.714179:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966987520:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1775764772.714209:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764772.716644:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24151 00000100:00100000:1.0:1775764772.716701:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24158 00000100:00100000:1.0:1775764772.716719:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c8700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:12041:x1862022978375680:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764772.716762:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b7692b0/0x36f9bdc9e4219ee2 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x64d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 12288->16383) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f1ecd expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 26 lru_type: 1 00010000:00010000:1.0:1775764772.716830:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764772.716836:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c8700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:12041:x1862022978375680:12345-192.168.202.146@tcp:104: Request processed in 127us (4059us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764772.716843:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24158 00010000:00010000:1.0:1775764772.716872:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764772.716878:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b7692b0/0x36f9bdc9e4219ee2 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x64d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 12288->16383) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f1ecd expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 26 lru_type: 1 00010000:00010000:1.0:1775764772.716886:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99702fc600 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:1.0:1775764772.716892:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b7692b0/0x36f9bdc9e4219ee2 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x64d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 12288->16383) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e5f1ecd expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 26 lru_type: 1 00010000:00010000:1.0:1775764772.716975:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b7692b0/0x36f9bdc9e4219ee2 lrc: 2/0,0 mode: --/PW res: [0x280000403:0x64d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 12288->16383) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f1ecd expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 26 lru_type: 1 00010000:00010000:1.0:1775764772.716983:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764772.717010:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b7692b0/0x36f9bdc9e4219ee2 lrc: 1/0,0 mode: --/PW res: [0x280000403:0x64d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 12288->16383) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f1ecd expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 26 lru_type: 1 00010000:00010000:1.0:1775764772.717018:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b7692b0/0x36f9bdc9e4219ee2 lrc: 0/0,0 mode: --/PW res: [0x280000403:0x64d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 12288->16383) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f1ecd expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 26 lru_type: 1 00010000:00010000:1.0:1775764772.717041:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764772.717042:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764772.717044:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764772.717074:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966987776:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1775764772.719214:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764772.722177:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24157 00000100:00100000:0.0:1775764772.736255:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764772.736457:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966987776:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764772.736475:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966987136:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764772.736482:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022966987008:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764772.736534:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978375936 00000100:00100000:1.0:1775764772.736540:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978375936 00000100:00100000:1.0:1775764772.736599:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24159 00000100:00100000:1.0:1775764772.736617:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c9500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:6580:x1862022978375936:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764772.736637:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000039e231d7/0x36f9bdc9e41ca3a7 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x3e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ed432 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 27 lru_type: 1 00010000:00010000:1.0:1775764772.736719:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764772.736726:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c9500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:6580:x1862022978375936:12345-192.168.202.146@tcp:104: Request processed in 110us (482us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764772.736733:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24159 00000100:00100000:2.0:1775764772.737256:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966985728:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.737269:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966985600:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1775764772.737774:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1775764772.737783:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000039e231d7/0x36f9bdc9e41ca3a7 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x3e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ed432 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 27 lru_type: 1 00010000:00010000:3.0:1775764772.737799:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9971227800 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:3.0:1775764772.737804:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000039e231d7/0x36f9bdc9e41ca3a7 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x3e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ed432 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 27 lru_type: 1 00010000:00010000:3.0:1775764772.737901:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000039e231d7/0x36f9bdc9e41ca3a7 lrc: 2/0,0 mode: --/PW res: [0x240000403:0x3e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed432 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 27 lru_type: 1 00010000:00010000:3.0:1775764772.737909:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764772.737940:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000039e231d7/0x36f9bdc9e41ca3a7 lrc: 1/0,0 mode: --/PW res: [0x240000403:0x3e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed432 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 27 lru_type: 1 00010000:00010000:3.0:1775764772.737949:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000039e231d7/0x36f9bdc9e41ca3a7 lrc: 0/0,0 mode: --/PW res: [0x240000403:0x3e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed432 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 27 lru_type: 1 00010000:00010000:3.0:1775764772.737972:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764772.737974:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764772.737976:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764772.738008:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966987264:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.738020:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966987520:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.738029:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966987392:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.738038:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966987648:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.738057:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966986496:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.738068:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efde300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966986368:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.738075:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966986624:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.738083:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966985984:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.738090:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966986240:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.738099:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966985856:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.738107:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966986112:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.738128:0:2382:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1775764772.738136:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966987904:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764772.748732:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764772.748825:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764772.748898:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:2.0:1775764772.748987:0:5564:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978376576 00000100:00100000:2.0:1775764772.749001:0:5564:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978376576 00000100:00100000:2.0:1775764772.749058:0:5564:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978376704 00000100:00100000:2.0:1775764772.749061:0:5564:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978376704 00000100:00100000:2.0:1775764772.749077:0:5564:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 1116 00000100:00100000:2.0:1775764772.749086:0:5564:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e6a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+5:6578:x1862022978376576:12345-192.168.202.146@tcp:104: 00010000:00010000:2.0:1775764772.749105:0:5564:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000025a0613b/0x36f9bdc9e42127f7 lrc: 2/0,0 mode: PR/PR res: [0x240000403:0x40:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5d9a4d expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:2.0:1775764772.749178:0:5564:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1775764772.749184:0:5564:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e6a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+5:6578:x1862022978376576:12345-192.168.202.146@tcp:104: Request processed in 100us (359us total) trans 0 rc 0/0 00000100:00100000:2.0:1775764772.749191:0:5564:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 1116 00000100:00100000:2.0:1775764772.749207:0:5564:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 1117 00000100:00100000:2.0:1775764772.749213:0:5564:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e5f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:16465:x1862022978376704:12345-192.168.202.146@tcp:104: 00010000:00010000:2.0:1775764772.749223:0:5564:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001e28c615/0x36f9bdc9e41ca8d9 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x4e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ee19f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 72 lru_type: 1 00010000:00010000:2.0:1775764772.749253:0:5564:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:2.0:1775764772.749258:0:5564:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e5f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:16465:x1862022978376704:12345-192.168.202.146@tcp:104: Request processed in 45us (360us total) trans 0 rc 0/0 00000100:00100000:2.0:1775764772.749263:0:5564:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 1117 00010000:00010000:2.0:1775764772.749279:0:5572:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1775764772.749299:0:5572:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000025a0613b/0x36f9bdc9e42127f7 lrc: 2/0,0 mode: PR/PR res: [0x240000403:0x40:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5d9a4d expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:2.0:1775764772.749307:0:5572:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99668bfa00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:2.0:1775764772.749313:0:5572:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000025a0613b/0x36f9bdc9e42127f7 lrc: 3/0,0 mode: PR/PR res: [0x240000403:0x40:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x428400010000 nid: local remote: 0x2aac6a50e5d9a4d expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:2.0:1775764772.749402:0:5572:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000025a0613b/0x36f9bdc9e42127f7 lrc: 2/0,0 mode: --/PR res: [0x240000403:0x40:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5d9a4d expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:2.0:1775764772.749410:0:5572:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764772.749431:0:5572:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000025a0613b/0x36f9bdc9e42127f7 lrc: 1/0,0 mode: --/PR res: [0x240000403:0x40:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5d9a4d expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:2.0:1775764772.749440:0:5572:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000025a0613b/0x36f9bdc9e42127f7 lrc: 0/0,0 mode: --/PR res: [0x240000403:0x40:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5d9a4d expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:2.0:1775764772.749460:0:5572:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764772.749464:0:5572:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001e28c615/0x36f9bdc9e41ca8d9 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x4e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ee19f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 72 lru_type: 1 00010000:00010000:2.0:1775764772.749470:0:5572:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970abec00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:2.0:1775764772.749473:0:5572:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001e28c615/0x36f9bdc9e41ca8d9 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x4e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ee19f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 72 lru_type: 1 00010000:00010000:2.0:1775764772.749504:0:5572:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001e28c615/0x36f9bdc9e41ca8d9 lrc: 2/0,0 mode: --/PW res: [0x240000403:0x4e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee19f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 72 lru_type: 1 00010000:00010000:2.0:1775764772.749511:0:5572:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764772.749517:0:5572:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001e28c615/0x36f9bdc9e41ca8d9 lrc: 1/0,0 mode: --/PW res: [0x240000403:0x4e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee19f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 72 lru_type: 1 00010000:00010000:2.0:1775764772.749524:0:5572:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001e28c615/0x36f9bdc9e41ca8d9 lrc: 0/0,0 mode: --/PW res: [0x240000403:0x4e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee19f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 72 lru_type: 1 00010000:00010000:2.0:1775764772.749536:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.749538:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.749540:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.749546:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764772.749560:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966988160:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.749624:0:2381:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1775764772.749630:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966988032:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764772.750855:0:11146:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1775764772.750874:0:11146:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978376448 00000100:00100000:0.0:1775764772.750880:0:11146:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978376448 00000100:00100000:0.0:1775764772.750935:0:11146:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24160 00000100:00100000:0.0:1775764772.750944:0:11146:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9966bf7850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+4:12040:x1862022978376448:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764772.750956:0:11146:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003557a767/0x36f9bdc9e41ca92d lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x4f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ee27f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 150 lru_type: 1 00010000:00010000:0.0:1775764772.751012:0:11146:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764772.751019:0:11146:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9966bf7850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+4:12040:x1862022978376448:12345-192.168.202.146@tcp:104: Request processed in 76us (2297us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764772.751025:0:11146:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24160 00010000:00010000:2.0:1775764772.751214:0:19800:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764772.751219:0:19800:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003557a767/0x36f9bdc9e41ca92d lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x4f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ee27f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 150 lru_type: 1 00010000:00010000:2.0:1775764772.751227:0:19800:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99700e5c00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:2.0:1775764772.751232:0:19800:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003557a767/0x36f9bdc9e41ca92d lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x4f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ee27f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 150 lru_type: 1 00010000:00010000:2.0:1775764772.752241:0:19800:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003557a767/0x36f9bdc9e41ca92d lrc: 2/0,0 mode: --/PW res: [0x240000403:0x4f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee27f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 150 lru_type: 1 00010000:00010000:2.0:1775764772.752259:0:19800:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:3.0:1775764772.752582:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966988288:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1775764772.755018:0:19800:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003557a767/0x36f9bdc9e41ca92d lrc: 1/0,0 mode: --/PW res: [0x240000403:0x4f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee27f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 150 lru_type: 1 00010000:00010000:2.0:1775764772.755035:0:19800:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003557a767/0x36f9bdc9e41ca92d lrc: 0/0,0 mode: --/PW res: [0x240000403:0x4f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee27f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 150 lru_type: 1 00010000:00010000:2.0:1775764772.755067:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.755069:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.755072:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764772.757243:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966987904:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.757714:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966988032:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.757724:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966988160:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764772.762510:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764772.762597:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764772.768855:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.776422:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978377088 00000100:00100000:0.0:1775764772.776439:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978377088 00000100:00100000:0.0:1775764772.776511:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978377216 00000100:00100000:0.0:1775764772.776514:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978377216 00000100:00100000:0.0:1775764772.776549:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24161 00000100:00100000:0.0:1775764772.776560:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e7b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:12039:x1862022978377088:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764772.776585:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000b2fa2735/0x36f9bdc9e41cb225 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x6b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef99a expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764772.776758:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764772.776766:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e7b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:12039:x1862022978377088:12345-192.168.202.146@tcp:104: Request processed in 208us (14265us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764772.776774:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24161 00000100:00100000:0.0:1775764772.776792:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24162 00000100:00100000:0.0:1775764772.776799:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e7480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:12196:x1862022978377216:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764772.776808:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000018bd3f91/0x36f9bdc9e41ca9f8 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x51:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ee4b6 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 104 lru_type: 1 00000100:00100000:1.0:1775764772.778361:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978377472 00000100:00100000:1.0:1775764772.778372:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978377472 00000100:00100000:1.0:1775764772.778410:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24163 00000100:00100000:1.0:1775764772.778419:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e7100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:12190:x1862022978377472:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764772.778433:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000996c885b/0x36f9bdc9e41cb1df lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x6a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef8e4 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764772.784479:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:2.0:1775764772.784495:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000b2fa2735/0x36f9bdc9e41cb225 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x6b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef99a expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764772.784510:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99702fd000 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:2.0:1775764772.784518:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000b2fa2735/0x36f9bdc9e41cb225 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x6b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ef99a expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764772.784629:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000b2fa2735/0x36f9bdc9e41cb225 lrc: 2/0,0 mode: --/PW res: [0x240000403:0x6b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef99a expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764772.784639:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764772.784666:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000b2fa2735/0x36f9bdc9e41cb225 lrc: 1/0,0 mode: --/PW res: [0x240000403:0x6b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef99a expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764772.784681:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000b2fa2735/0x36f9bdc9e41cb225 lrc: 0/0,0 mode: --/PW res: [0x240000403:0x6b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef99a expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764772.784716:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1775764772.784720:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000018bd3f91/0x36f9bdc9e41ca9f8 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x51:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ee4b6 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 104 lru_type: 1 00010000:00010000:2.0:1775764772.784727:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970abe200 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:2.0:1775764772.784732:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000018bd3f91/0x36f9bdc9e41ca9f8 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x51:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ee4b6 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 104 lru_type: 1 00010000:00010000:2.0:1775764772.784787:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000018bd3f91/0x36f9bdc9e41ca9f8 lrc: 2/0,0 mode: --/PW res: [0x240000403:0x51:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee4b6 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 104 lru_type: 1 00010000:00010000:2.0:1775764772.784795:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764772.784802:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000018bd3f91/0x36f9bdc9e41ca9f8 lrc: 1/0,0 mode: --/PW res: [0x240000403:0x51:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee4b6 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 104 lru_type: 1 00010000:00010000:2.0:1775764772.784809:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000018bd3f91/0x36f9bdc9e41ca9f8 lrc: 0/0,0 mode: --/PW res: [0x240000403:0x51:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee4b6 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 104 lru_type: 1 00010000:00010000:2.0:1775764772.784823:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764772.784826:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000996c885b/0x36f9bdc9e41cb1df lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x6a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef8e4 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764772.784839:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99413d2c00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:2.0:1775764772.784843:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000996c885b/0x36f9bdc9e41cb1df lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x6a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ef8e4 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764772.784874:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000996c885b/0x36f9bdc9e41cb1df lrc: 2/0,0 mode: --/PW res: [0x240000403:0x6a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef8e4 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764772.784881:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764772.784886:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000996c885b/0x36f9bdc9e41cb1df lrc: 1/0,0 mode: --/PW res: [0x240000403:0x6a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef8e4 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764772.784893:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000996c885b/0x36f9bdc9e41cb1df lrc: 0/0,0 mode: --/PW res: [0x240000403:0x6a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef8e4 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764772.784906:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.784908:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.784910:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764772.784952:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966988544:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.785057:0:2382:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:2.0:1775764772.785064:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966988416:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.785089:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966988672:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1775764772.786053:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/0 locks to priority blp list, 0 blwis in pool 00000100:00100000:0.0:1775764772.786068:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e7480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:12196:x1862022978377216:12345-192.168.202.146@tcp:104: Request processed in 9266us (23467us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764772.786079:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24162 00010000:00010000:3.0:1775764772.786234:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764772.786647:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/0 locks to priority blp list, 0 blwis in pool 00000100:00100000:1.0:1775764772.786657:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e7100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:12190:x1862022978377472:12345-192.168.202.146@tcp:104: Request processed in 8237us (17811us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764772.786664:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24163 00000100:00100000:1.0:1775764772.786760:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764772.786784:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764772.786801:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764772.786820:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764772.786858:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764772.786889:0:5562:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1775764772.786907:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978377728 00000100:00100000:1.0:1775764772.786914:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978377728 00000100:00100000:1.0:1775764772.786954:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978378880 00000100:00100000:1.0:1775764772.786957:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978378880 00000100:00100000:1.0:1775764772.786967:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978379008 00000100:00100000:1.0:1775764772.786969:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978379008 00000100:00100000:1.0:1775764772.786977:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978379136 00000100:00100000:1.0:1775764772.786979:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978379136 00000100:00100000:1.0:1775764772.786990:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24164 00000100:00100000:1.0:1775764772.786999:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e6680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:12187:x1862022978377728:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764772.787013:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000095a27c5b/0x36f9bdc9e41caa3e lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x52:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ee56c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 129 lru_type: 1 00000100:00100000:2.0:1775764772.787213:0:5564:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:2.0:1775764772.787225:0:5564:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978379264 00000100:00100000:2.0:1775764772.787228:0:5564:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978379264 00000100:00100000:2.0:1775764772.787252:0:5564:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 1118 00000100:00100000:2.0:1775764772.787259:0:5564:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0ccc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+7:12188:x1862022978379264:12345-192.168.202.146@tcp:104: 00010000:00010000:2.0:1775764772.787268:0:5564:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c11d69a0/0x36f9bdc9e41ca3f4 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x3e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ed520 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 29 lru_type: 1 00010000:00010000:3.0:1775764772.787344:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764772.787385:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966986880:192.168.202.146@tcp:35:rm.0 00000002:00100000:3.0:1775764772.787412:0:19896:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f994f2e4e00 x1862022950356352/t4294969644(4294969644) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764071 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00010000:00010000:3.0:1775764772.787481:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eac00/0x36f9bdc9e421c6d5 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5faa17 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764772.787491:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970abf200/0x36f9bdc9e41caa45 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x11/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ee588 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 102 lru_type: 1 00010000:00010000:3.0:1775764772.787499:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970abfa00/0x36f9bdc9e41ca9ff lrc: 1/0,0 mode: --/CR res: [0x200000401:0x2b2:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ee4c4 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 241 lru_type: 1 00010000:00010000:3.0:1775764772.787504:0:19896:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:3.0:1775764772.787507:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eac00/0x36f9bdc9e421c6d5 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5faa17 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764772.787517:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970abf200/0x36f9bdc9e41caa45 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2b2:0x0].0x0 bits 0x11/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ee588 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 102 lru_type: 1 00010000:00010000:3.0:1775764772.787524:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970abfa00/0x36f9bdc9e41ca9ff lrc: 0/0,0 mode: --/CR res: [0x200000401:0x2b2:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ee4c4 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 241 lru_type: 1 00000020:00100000:3.0:1775764772.787543:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1775764772.787549:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970eded80 x1862022966988928/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764772.787566:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966988928:192.168.202.146@tcp:36:rm.0 00000100:00100000:0.0:1775764772.787649:0:11131:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24165 00000100:00100000:0.0:1775764772.787659:0:11131:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e7800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+7:12182:x1862022978378880:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764772.787669:0:11131:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000002144360d/0x36f9bdc9e41ca441 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x3f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ed5c1 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764772.787805:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764772.787812:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e6680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:12187:x1862022978377728:12345-192.168.202.146@tcp:104: Request processed in 814us (1052us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764772.787820:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24164 00000100:00100000:1.0:1775764772.787837:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24166 00000100:00100000:1.0:1775764772.787843:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e5c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+7:6580:x1862022978379008:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764772.787851:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000acba8da0/0x36f9bdc9e4219080 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x640:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5ef074 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:2.0:1775764772.790974:0:5564:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:2.0:1775764772.790993:0:5564:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0ccc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+7:12188:x1862022978379264:12345-192.168.202.146@tcp:104: Request processed in 3729us (4128us total) trans 0 rc 0/0 00000100:00100000:2.0:1775764772.791005:0:5564:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 1118 00010000:00010000:2.0:1775764772.791076:0:11210:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1775764772.791083:0:11210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000095a27c5b/0x36f9bdc9e41caa3e lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x52:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ee56c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 129 lru_type: 1 00010000:00010000:2.0:1775764772.791095:0:11210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970abfe00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:2.0:1775764772.791101:0:11210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000095a27c5b/0x36f9bdc9e41caa3e lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x52:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ee56c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 129 lru_type: 1 00010000:00010000:2.0:1775764772.791184:0:11210:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000095a27c5b/0x36f9bdc9e41caa3e lrc: 2/0,0 mode: --/PW res: [0x240000403:0x52:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee56c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 129 lru_type: 1 00010000:00010000:2.0:1775764772.791191:0:11210:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764772.791221:0:11210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000095a27c5b/0x36f9bdc9e41caa3e lrc: 1/0,0 mode: --/PW res: [0x240000403:0x52:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee56c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 129 lru_type: 1 00010000:00010000:2.0:1775764772.791229:0:11210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000095a27c5b/0x36f9bdc9e41caa3e lrc: 0/0,0 mode: --/PW res: [0x240000403:0x52:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee56c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 129 lru_type: 1 00010000:00010000:2.0:1775764772.791255:0:11210:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764772.791259:0:11210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c11d69a0/0x36f9bdc9e41ca3f4 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x3e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ed520 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 29 lru_type: 1 00010000:00010000:2.0:1775764772.791265:0:11210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f994f3b8200 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:2.0:1775764772.791268:0:11210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c11d69a0/0x36f9bdc9e41ca3f4 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x3e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ed520 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 29 lru_type: 1 00010000:00010000:2.0:1775764772.791310:0:11210:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c11d69a0/0x36f9bdc9e41ca3f4 lrc: 2/0,0 mode: --/PW res: [0x280000403:0x3e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed520 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 29 lru_type: 1 00010000:00010000:2.0:1775764772.791523:0:11210:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764772.791531:0:11210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c11d69a0/0x36f9bdc9e41ca3f4 lrc: 1/0,0 mode: --/PW res: [0x280000403:0x3e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed520 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 29 lru_type: 1 00010000:00010000:2.0:1775764772.791539:0:11210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c11d69a0/0x36f9bdc9e41ca3f4 lrc: 0/0,0 mode: --/PW res: [0x280000403:0x3e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed520 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 29 lru_type: 1 00010000:00010000:2.0:1775764772.791555:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.791557:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.791558:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.791565:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764772.791610:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966989056:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.791681:0:2382:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1775764772.791689:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efde300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966989184:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.791856:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413d80 going to sleep for 11 seconds 00010000:00010000:0.0:1775764772.792063:0:11131:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764772.792072:0:11131:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e7800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+6:12182:x1862022978378880:12345-192.168.202.146@tcp:104: Request processed in 4415us (5285us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764772.792081:0:11131:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24165 00000100:00100000:0.0:1775764772.792103:0:11131:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24167 00000100:00100000:0.0:1775764772.792115:0:11131:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9966bf6850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:12181:x1862022978379136:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764772.792127:0:11131:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c29fe1cb/0x36f9bdc9e421a2e7 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x650:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f2bd1 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00010000:00010000:0.0:1775764772.792172:0:11131:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:0.0:1775764772.792178:0:11131:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9966bf6850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:12181:x1862022978379136:12345-192.168.202.146@tcp:104: Request processed in 68us (5357us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764772.792184:0:11131:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24167 00010000:00010000:3.0:1775764772.792307:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1775764772.792313:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000002144360d/0x36f9bdc9e41ca441 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x3f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ed5c1 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1775764772.792330:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9971227600 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:3.0:1775764772.792335:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000002144360d/0x36f9bdc9e41ca441 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x3f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ed5c1 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1775764772.792399:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000002144360d/0x36f9bdc9e41ca441 lrc: 2/0,0 mode: --/PW res: [0x280000403:0x3f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed5c1 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1775764772.792406:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764772.792421:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000002144360d/0x36f9bdc9e41ca441 lrc: 1/0,0 mode: --/PW res: [0x280000403:0x3f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed5c1 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1775764772.792429:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000002144360d/0x36f9bdc9e41ca441 lrc: 0/0,0 mode: --/PW res: [0x280000403:0x3f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed5c1 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1775764772.792443:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1775764772.792446:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c29fe1cb/0x36f9bdc9e421a2e7 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x650:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f2bd1 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00010000:00010000:3.0:1775764772.792453:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9971070e00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:3.0:1775764772.792456:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c29fe1cb/0x36f9bdc9e421a2e7 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x650:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e5f2bd1 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00010000:00010000:3.0:1775764772.792488:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c29fe1cb/0x36f9bdc9e421a2e7 lrc: 2/0,0 mode: --/PW res: [0x280000403:0x650:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f2bd1 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00010000:00010000:3.0:1775764772.792495:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764772.792501:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c29fe1cb/0x36f9bdc9e421a2e7 lrc: 1/0,0 mode: --/PW res: [0x280000403:0x650:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f2bd1 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00010000:00010000:3.0:1775764772.792508:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c29fe1cb/0x36f9bdc9e421a2e7 lrc: 0/0,0 mode: --/PW res: [0x280000403:0x650:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f2bd1 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00010000:00010000:3.0:1775764772.792521:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764772.792522:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764772.792524:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764772.792535:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966989440:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.792562:0:2381:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1775764772.792567:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966989312:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1775764772.792776:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764772.795270:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.795736:0:11131:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978379520 00000100:00100000:0.0:1775764772.795773:0:11131:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978379520 00000100:00100000:0.0:1775764772.795823:0:11131:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24168 00000100:00100000:0.0:1775764772.795832:0:11131:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c8380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:8216:x1862022978379520:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764772.795849:0:11131:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000004c619a3d/0x36f9bdc9e4219286 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x641:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5ef72e expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:0.0:1775764772.795909:0:11131:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764772.795915:0:11131:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c8380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:8216:x1862022978379520:12345-192.168.202.146@tcp:104: Request processed in 86us (654us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764772.795923:0:11131:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24168 00010000:00010000:3.0:1775764772.796027:0:11138:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1775764772.796034:0:11138:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000004c619a3d/0x36f9bdc9e4219286 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x641:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5ef72e expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:3.0:1775764772.796043:0:11138:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f994ed8c800 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:3.0:1775764772.796059:0:11138:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000004c619a3d/0x36f9bdc9e4219286 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x641:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e5ef72e expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:3.0:1775764772.796142:0:11138:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000004c619a3d/0x36f9bdc9e4219286 lrc: 2/0,0 mode: --/PW res: [0x280000403:0x641:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5ef72e expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:3.0:1775764772.796149:0:11138:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764772.796175:0:11138:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000004c619a3d/0x36f9bdc9e4219286 lrc: 1/0,0 mode: --/PW res: [0x280000403:0x641:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5ef72e expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:3.0:1775764772.796184:0:11138:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000004c619a3d/0x36f9bdc9e4219286 lrc: 0/0,0 mode: --/PW res: [0x280000403:0x641:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5ef72e expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:3.0:1775764772.796204:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764772.796206:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764772.796209:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764772.796221:0:2381:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1775764772.796235:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966989568:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764772.796653:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.796673:0:11146:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978379648 00000100:00100000:0.0:1775764772.796678:0:11146:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978379648 00000100:00100000:0.0:1775764772.796703:0:11131:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24169 00000100:00100000:0.0:1775764772.796710:0:11131:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c9180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:12041:x1862022978379648:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764772.796721:0:11131:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005cfce96d/0x36f9bdc9e41ca487 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x40:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ed669 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 0 00010000:00010000:0.0:1775764772.796780:0:11131:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764772.796785:0:11131:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c9180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:12041:x1862022978379648:12345-192.168.202.146@tcp:104: Request processed in 76us (134us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764772.796791:0:11131:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24169 00000100:00100000:0.0:1775764772.796833:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.796846:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978379776 00000100:00100000:0.0:1775764772.796849:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978379776 00000100:00100000:0.0:1775764772.796863:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24170 00000100:00100000:0.0:1775764772.796869:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0ca300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:12186:x1862022978379776:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764772.796876:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000cf9baea5/0x36f9bdc9e41ca4f7 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x42:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ed788 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 0 00010000:00010000:0.0:1775764772.796900:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:0.0:1775764772.796905:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0ca300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:12186:x1862022978379776:12345-192.168.202.146@tcp:104: Request processed in 36us (72us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764772.796911:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24170 00010000:00010000:2.0:1775764772.797017:0:11210:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1775764772.797023:0:11210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005cfce96d/0x36f9bdc9e41ca487 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x40:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ed669 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 0 00010000:00010000:2.0:1775764772.797031:0:11210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f994f3b9400 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:2.0:1775764772.797035:0:11210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005cfce96d/0x36f9bdc9e41ca487 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x40:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ed669 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 0 00010000:00010000:2.0:1775764772.797098:0:11210:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005cfce96d/0x36f9bdc9e41ca487 lrc: 2/0,0 mode: --/PW res: [0x280000403:0x40:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed669 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 0 00010000:00010000:2.0:1775764772.797105:0:11210:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764772.797118:0:11210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005cfce96d/0x36f9bdc9e41ca487 lrc: 1/0,0 mode: --/PW res: [0x280000403:0x40:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed669 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 0 00010000:00010000:2.0:1775764772.797126:0:11210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005cfce96d/0x36f9bdc9e41ca487 lrc: 0/0,0 mode: --/PW res: [0x280000403:0x40:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed669 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 0 00010000:00010000:2.0:1775764772.797141:0:11210:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764772.797144:0:11210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000cf9baea5/0x36f9bdc9e41ca4f7 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x42:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ed788 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 0 00010000:00010000:2.0:1775764772.797150:0:11210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9971227000 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:2.0:1775764772.797153:0:11210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000cf9baea5/0x36f9bdc9e41ca4f7 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x42:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ed788 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 0 00010000:00010000:2.0:1775764772.797185:0:11210:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000cf9baea5/0x36f9bdc9e41ca4f7 lrc: 2/0,0 mode: --/PW res: [0x280000403:0x42:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed788 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 0 00010000:00010000:2.0:1775764772.797191:0:11210:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764772.797196:0:11210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000cf9baea5/0x36f9bdc9e41ca4f7 lrc: 1/0,0 mode: --/PW res: [0x280000403:0x42:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed788 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 0 00010000:00010000:2.0:1775764772.797202:0:11210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000cf9baea5/0x36f9bdc9e41ca4f7 lrc: 0/0,0 mode: --/PW res: [0x280000403:0x42:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed788 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 0 00010000:00010000:2.0:1775764772.797214:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.797215:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.797217:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764772.797226:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966989824:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.797259:0:2382:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1775764772.797264:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966989696:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764772.797490:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.797507:0:11146:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978379392 00000100:00100000:0.0:1775764772.797511:0:11146:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978379392 00000100:00100000:0.0:1775764772.797529:0:11146:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24171 00000100:00100000:0.0:1775764772.797535:0:11146:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c8700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+5:12180:x1862022978379392:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764772.797543:0:11146:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000007719768c/0x36f9bdc9e41ca4b8 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x41:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ed6ee expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 0 00010000:00010000:0.0:1775764772.797574:0:11146:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764772.797580:0:11146:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c8700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+5:12180:x1862022978379392:12345-192.168.202.146@tcp:104: Request processed in 45us (91us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764772.797586:0:11146:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24171 00000100:00100000:0.0:1775764772.797660:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.797671:0:11131:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978380160 00000100:00100000:0.0:1775764772.797674:0:11131:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978380160 00000100:00100000:0.0:1775764772.797689:0:11131:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24172 00000100:00100000:0.0:1775764772.797694:0:11131:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c8000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:12051:x1862022978380160:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764772.797702:0:11131:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f01cb6c5/0x36f9bdc9e41ca6ef lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x48:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4edcb3 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 42 lru_type: 1 00010000:00010000:0.0:1775764772.797724:0:11131:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:0.0:1775764772.797728:0:11131:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c8000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:12051:x1862022978380160:12345-192.168.202.146@tcp:104: Request processed in 34us (69us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764772.797734:0:11131:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24172 00000100:00100000:0.0:1775764772.797792:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.797803:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978380288 00000100:00100000:0.0:1775764772.797805:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978380288 00000100:00100000:0.0:1775764772.797819:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24173 00000100:00100000:0.0:1775764772.797824:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c9500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:12040:x1862022978380288:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764772.797832:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000cc4b2477/0x36f9bdc9e41ca735 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x49:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4edd5b expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 57 lru_type: 1 00010000:00010000:0.0:1775764772.797851:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:0.0:1775764772.797855:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c9500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:12040:x1862022978380288:12345-192.168.202.146@tcp:104: Request processed in 32us (64us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764772.797861:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24173 00000100:00100000:0.0:1775764772.797897:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.797914:0:11146:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/3 left), rc = 0 00000100:00100000:0.0:1775764772.797923:0:11146:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978380672 00010000:00010000:3.0:1775764772.798524:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:3.0:1775764772.798531:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000007719768c/0x36f9bdc9e41ca4b8 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x41:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ed6ee expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 0 00010000:00010000:3.0:1775764772.798539:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f994f3b8e00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:3.0:1775764772.798544:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000007719768c/0x36f9bdc9e41ca4b8 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x41:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ed6ee expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 0 00010000:00010000:3.0:1775764772.798618:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000007719768c/0x36f9bdc9e41ca4b8 lrc: 2/0,0 mode: --/PW res: [0x280000403:0x41:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed6ee expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 0 00010000:00010000:3.0:1775764772.798625:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764772.798636:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000007719768c/0x36f9bdc9e41ca4b8 lrc: 1/0,0 mode: --/PW res: [0x280000403:0x41:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed6ee expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 0 00010000:00010000:3.0:1775764772.798643:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000007719768c/0x36f9bdc9e41ca4b8 lrc: 0/0,0 mode: --/PW res: [0x280000403:0x41:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed6ee expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 0 00010000:00010000:3.0:1775764772.798659:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1775764772.798662:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f01cb6c5/0x36f9bdc9e41ca6ef lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x48:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4edcb3 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 42 lru_type: 1 00010000:00010000:3.0:1775764772.798668:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f994f3d9a00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:3.0:1775764772.798672:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f01cb6c5/0x36f9bdc9e41ca6ef lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x48:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4edcb3 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 42 lru_type: 1 00010000:00010000:3.0:1775764772.798701:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f01cb6c5/0x36f9bdc9e41ca6ef lrc: 2/0,0 mode: --/PW res: [0x240000403:0x48:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4edcb3 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 42 lru_type: 1 00010000:00010000:3.0:1775764772.798707:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764772.798712:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f01cb6c5/0x36f9bdc9e41ca6ef lrc: 1/0,0 mode: --/PW res: [0x240000403:0x48:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4edcb3 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 42 lru_type: 1 00010000:00010000:3.0:1775764772.798719:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f01cb6c5/0x36f9bdc9e41ca6ef lrc: 0/0,0 mode: --/PW res: [0x240000403:0x48:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4edcb3 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 42 lru_type: 1 00010000:00010000:3.0:1775764772.798730:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1775764772.798733:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000cc4b2477/0x36f9bdc9e41ca735 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x49:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4edd5b expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 57 lru_type: 1 00010000:00010000:3.0:1775764772.798751:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f996695cc00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:3.0:1775764772.798754:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000cc4b2477/0x36f9bdc9e41ca735 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x49:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4edd5b expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 57 lru_type: 1 00010000:00010000:3.0:1775764772.798781:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000cc4b2477/0x36f9bdc9e41ca735 lrc: 2/0,0 mode: --/PW res: [0x240000403:0x49:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4edd5b expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 57 lru_type: 1 00010000:00010000:3.0:1775764772.798787:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764772.798791:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000cc4b2477/0x36f9bdc9e41ca735 lrc: 1/0,0 mode: --/PW res: [0x240000403:0x49:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4edd5b expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 57 lru_type: 1 00010000:00010000:3.0:1775764772.798806:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000cc4b2477/0x36f9bdc9e41ca735 lrc: 0/0,0 mode: --/PW res: [0x240000403:0x49:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4edd5b expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 57 lru_type: 1 00010000:00010000:3.0:1775764772.798818:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764772.798820:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764772.798821:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764772.798838:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966989952:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.798877:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966990208:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.798903:0:2381:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1775764772.798908:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966990080:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1775764772.799000:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.799088:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.799094:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764772.799128:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966988288:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764772.800450:0:11146:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978380672 00000100:00100000:0.0:1775764772.800534:0:11146:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24174 00000100:00100000:0.0:1775764772.800545:0:11146:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970e0ac50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+4:12193:x1862022978380672:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764772.800566:0:11146:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000609c9638/0x36f9bdc9e41ca7ba lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x4b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ede9d expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 58 lru_type: 1 00010000:00010000:0.0:1775764772.800651:0:11146:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764772.800658:0:11146:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970e0ac50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:LOV_OSC_UUID+4:12193:x1862022978380672:12345-192.168.202.146@tcp:104: Request processed in 114us (2760us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764772.800665:0:11146:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24174 00000100:00100000:0.0:1775764772.800934:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.800960:0:11131:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978380800 00000100:00100000:0.0:1775764772.800963:0:11131:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978380800 00000100:00100000:0.0:1775764772.801000:0:11131:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24175 00000100:00100000:0.0:1775764772.801007:0:11131:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c8380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:12215:x1862022978380800:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764772.801016:0:11131:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000009eadeed7/0x36f9bdc9e41ca807 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x4c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4edf53 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 58 lru_type: 1 00010000:00010000:0.0:1775764772.801042:0:11131:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:0.0:1775764772.801046:0:11131:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c8380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:12215:x1862022978380800:12345-192.168.202.146@tcp:104: Request processed in 41us (115us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764772.801051:0:11131:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24175 00010000:00010000:1.0:1775764772.802236:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:1.0:1775764772.802248:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e5c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:6580:x1862022978379008:12345-192.168.202.146@tcp:104: Request processed in 14403us (15443us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764772.802257:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24166 00010000:00010000:3.0:1775764772.802444:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:3.0:1775764772.802453:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000609c9638/0x36f9bdc9e41ca7ba lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x4b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ede9d expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 58 lru_type: 1 00010000:00010000:3.0:1775764772.802463:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f996695d400 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:3.0:1775764772.802471:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000609c9638/0x36f9bdc9e41ca7ba lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x4b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ede9d expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 58 lru_type: 1 00010000:00010000:3.0:1775764772.802562:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000609c9638/0x36f9bdc9e41ca7ba lrc: 2/0,0 mode: --/PW res: [0x240000403:0x4b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ede9d expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 58 lru_type: 1 00010000:00010000:3.0:1775764772.802569:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764772.802589:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000609c9638/0x36f9bdc9e41ca7ba lrc: 1/0,0 mode: --/PW res: [0x240000403:0x4b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ede9d expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 58 lru_type: 1 00010000:00010000:3.0:1775764772.802597:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000609c9638/0x36f9bdc9e41ca7ba lrc: 0/0,0 mode: --/PW res: [0x240000403:0x4b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ede9d expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 58 lru_type: 1 00010000:00010000:3.0:1775764772.802630:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1775764772.802633:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000009eadeed7/0x36f9bdc9e41ca807 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x4c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4edf53 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 58 lru_type: 1 00010000:00010000:3.0:1775764772.802639:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f996695d600 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:3.0:1775764772.802642:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000009eadeed7/0x36f9bdc9e41ca807 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x4c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4edf53 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 58 lru_type: 1 00010000:00010000:3.0:1775764772.802670:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000009eadeed7/0x36f9bdc9e41ca807 lrc: 2/0,0 mode: --/PW res: [0x240000403:0x4c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4edf53 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 58 lru_type: 1 00010000:00010000:3.0:1775764772.802677:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764772.802683:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000009eadeed7/0x36f9bdc9e41ca807 lrc: 1/0,0 mode: --/PW res: [0x240000403:0x4c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4edf53 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 58 lru_type: 1 00010000:00010000:3.0:1775764772.802690:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000009eadeed7/0x36f9bdc9e41ca807 lrc: 0/0,0 mode: --/PW res: [0x240000403:0x4c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4edf53 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 58 lru_type: 1 00010000:00010000:3.0:1775764772.802701:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1775764772.802704:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000acba8da0/0x36f9bdc9e4219080 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x640:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5ef074 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:3.0:1775764772.802715:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9966d48e00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:3.0:1775764772.802719:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000acba8da0/0x36f9bdc9e4219080 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x640:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e5ef074 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:3.0:1775764772.802767:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000acba8da0/0x36f9bdc9e4219080 lrc: 2/0,0 mode: --/PW res: [0x280000403:0x640:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5ef074 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:3.0:1775764772.802774:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764772.802780:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000acba8da0/0x36f9bdc9e4219080 lrc: 1/0,0 mode: --/PW res: [0x280000403:0x640:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5ef074 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:3.0:1775764772.802786:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000acba8da0/0x36f9bdc9e4219080 lrc: 0/0,0 mode: --/PW res: [0x280000403:0x640:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5ef074 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:3.0:1775764772.802801:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764772.802802:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764772.802804:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764772.802820:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966990464:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.802886:0:2381:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1775764772.802892:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966990336:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.802912:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966990592:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1775764772.804209:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.804222:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764772.821139:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966989696:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.821155:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966989824:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.821167:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efde300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966989184:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.821178:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966989056:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.821186:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966988416:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.821194:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966988672:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.821202:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966988544:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764772.823938:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966988928:192.168.202.146@tcp:36:rm.0 00000080:00200000:0.0:1775764772.823989:0:19896:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2b2:0x0](ffff8f9952d8ba88) 00000080:00200000:0.0:1775764772.824102:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764772.824115:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=WORD, flags=0 00000080:00200008:0.0:1775764772.824126:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file WORD:[0x200000401:0x277:0x0](ffff8f9952e64b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764772.824131:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x277:0x0](ffff8f9952e64b08),name=WORD 00000002:00010000:0.0:1775764772.824151:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x277:0x0]) in obj [0x200000401:0x26c:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764772.824164:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/631 (0 0) 00010000:00010000:0.0:1775764772.824213:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c6dc lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764772.824223:0:19896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c6dc lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764772.824231:0:19896:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c6dc lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764772.824251:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c9180 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966990720:192.168.202.146@tcp:101:rm.0 00000100:00100000:0.0:1775764772.826354:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f600 going to sleep for 11 seconds 00000100:00100000:3.0:1775764772.827247:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966990336:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.827264:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966990592:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.827275:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966990464:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.827297:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966990080:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.827308:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966989952:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.827328:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966990208:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.827337:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966989568:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.827348:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966989312:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764772.829968:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.830008:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.830038:0:11131:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978381184 00000100:00100000:0.0:1775764772.830058:0:11131:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978381184 00000100:00100000:0.0:1775764772.830134:0:11131:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978381312 00000100:00100000:0.0:1775764772.830138:0:11131:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978381312 00000100:00100000:0.0:1775764772.830156:0:11131:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24176 00000100:00100000:0.0:1775764772.830164:0:11131:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0ca300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:6579:x1862022978381184:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764772.830184:0:11131:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c6ec0b7f/0x36f9bdc9e41ca84d lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x4d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4edffb expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 71 lru_type: 1 00010000:00010000:0.0:1775764772.830262:0:11131:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764772.830269:0:11131:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0ca300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:6579:x1862022978381184:12345-192.168.202.146@tcp:104: Request processed in 106us (311us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764772.830276:0:11131:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24176 00000100:00100000:0.0:1775764772.830293:0:11131:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24177 00000100:00100000:0.0:1775764772.830299:0:11131:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c8700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:6578:x1862022978381312:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764772.830306:0:11131:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001cab28fe/0x36f9bdc9e41ca5de lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x44:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eda36 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 93 lru_type: 1 00010000:00010000:0.0:1775764772.830351:0:11131:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:0.0:1775764772.830356:0:11131:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c8700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:6578:x1862022978381312:12345-192.168.202.146@tcp:104: Request processed in 58us (348us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764772.830362:0:11131:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24177 00010000:00010000:2.0:1775764772.830546:0:5572:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1775764772.830553:0:5572:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c6ec0b7f/0x36f9bdc9e41ca84d lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x4d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4edffb expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 71 lru_type: 1 00010000:00010000:2.0:1775764772.830562:0:5572:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f996695ca00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:2.0:1775764772.830567:0:5572:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c6ec0b7f/0x36f9bdc9e41ca84d lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x4d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4edffb expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 71 lru_type: 1 00010000:00010000:2.0:1775764772.830695:0:5572:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c6ec0b7f/0x36f9bdc9e41ca84d lrc: 2/0,0 mode: --/PW res: [0x240000403:0x4d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4edffb expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 71 lru_type: 1 00010000:00010000:2.0:1775764772.830703:0:5572:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764772.830726:0:5572:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c6ec0b7f/0x36f9bdc9e41ca84d lrc: 1/0,0 mode: --/PW res: [0x240000403:0x4d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4edffb expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 71 lru_type: 1 00010000:00010000:2.0:1775764772.830733:0:5572:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c6ec0b7f/0x36f9bdc9e41ca84d lrc: 0/0,0 mode: --/PW res: [0x240000403:0x4d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4edffb expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 71 lru_type: 1 00010000:00010000:2.0:1775764772.830785:0:5572:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764772.830788:0:5572:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001cab28fe/0x36f9bdc9e41ca5de lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x44:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eda36 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 93 lru_type: 1 00010000:00010000:2.0:1775764772.830794:0:5572:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9971079800 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:2.0:1775764772.830796:0:5572:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001cab28fe/0x36f9bdc9e41ca5de lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x44:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4eda36 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 93 lru_type: 1 00010000:00010000:2.0:1775764772.830828:0:5572:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001cab28fe/0x36f9bdc9e41ca5de lrc: 2/0,0 mode: --/PW res: [0x240000403:0x44:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eda36 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 93 lru_type: 1 00010000:00010000:2.0:1775764772.830835:0:5572:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764772.830841:0:5572:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001cab28fe/0x36f9bdc9e41ca5de lrc: 1/0,0 mode: --/PW res: [0x240000403:0x44:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eda36 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 93 lru_type: 1 00010000:00010000:2.0:1775764772.830847:0:5572:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001cab28fe/0x36f9bdc9e41ca5de lrc: 0/0,0 mode: --/PW res: [0x240000403:0x44:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eda36 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 93 lru_type: 1 00010000:00010000:2.0:1775764772.830859:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.830860:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.830862:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764772.830878:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966990976:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.830937:0:2382:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1775764772.830943:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966990848:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1775764772.831127:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764772.834847:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.834878:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.834900:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.834923:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.834957:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.834985:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:3.0:1775764772.835158:0:5563:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978382592 00000100:00100000:3.0:1775764772.835167:0:5563:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978382592 00000100:00100000:3.0:1775764772.835210:0:5563:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978382848 00000100:00100000:3.0:1775764772.835213:0:5563:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978382848 00000100:00100000:3.0:1775764772.835225:0:5563:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 1119 00000100:00100000:3.0:1775764772.835234:0:5563:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0ca680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+5:16465:x1862022978382592:12345-192.168.202.146@tcp:104: 00010000:00010000:3.0:1775764772.835252:0:5563:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000b2d6c772/0x36f9bdc9e41cb272 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x6b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4efa6c expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764772.835321:0:5563:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1775764772.835328:0:5563:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0ca680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+5:16465:x1862022978382592:12345-192.168.202.146@tcp:104: Request processed in 95us (369us total) trans 0 rc 0/0 00000100:00100000:3.0:1775764772.835335:0:5563:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 1119 00010000:00010000:3.0:1775764772.835354:0:19800:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1775764772.835360:0:19800:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000b2d6c772/0x36f9bdc9e41cb272 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x6b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4efa6c expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764772.835387:0:19800:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f994f080600 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:3.0:1775764772.835392:0:19800:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000b2d6c772/0x36f9bdc9e41cb272 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x6b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4efa6c expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764772.835473:0:19800:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000b2d6c772/0x36f9bdc9e41cb272 lrc: 2/0,0 mode: --/PW res: [0x280000403:0x6b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4efa6c expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764772.835480:0:19800:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764772.835498:0:19800:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000b2d6c772/0x36f9bdc9e41cb272 lrc: 1/0,0 mode: --/PW res: [0x280000403:0x6b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4efa6c expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764772.835506:0:19800:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000b2d6c772/0x36f9bdc9e41cb272 lrc: 0/0,0 mode: --/PW res: [0x280000403:0x6b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4efa6c expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764772.835527:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764772.835529:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764772.835530:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764772.835551:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966991104:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.835627:0:5563:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 1120 00000100:00100000:3.0:1775764772.835635:0:5563:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0cad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:8216:x1862022978382848:12345-192.168.202.146@tcp:104: 00010000:00010000:3.0:1775764772.835643:0:5563:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c5083c7f/0x36f9bdc9e421ab29 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x656:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f459a expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:3.0:1775764772.835671:0:5563:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1775764772.835676:0:5563:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0cad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:8216:x1862022978382848:12345-192.168.202.146@tcp:104: Request processed in 43us (691us total) trans 0 rc 0/0 00000100:00100000:3.0:1775764772.835682:0:5563:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 1120 00010000:00010000:3.0:1775764772.835693:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1775764772.835698:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c5083c7f/0x36f9bdc9e421ab29 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x656:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f459a expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:3.0:1775764772.835705:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99702fc200 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:3.0:1775764772.835709:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c5083c7f/0x36f9bdc9e421ab29 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x656:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e5f459a expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:3.0:1775764772.835774:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c5083c7f/0x36f9bdc9e421ab29 lrc: 2/0,0 mode: --/PW res: [0x280000403:0x656:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f459a expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:3.0:1775764772.835782:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764772.835793:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c5083c7f/0x36f9bdc9e421ab29 lrc: 1/0,0 mode: --/PW res: [0x280000403:0x656:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f459a expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:3.0:1775764772.835800:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c5083c7f/0x36f9bdc9e421ab29 lrc: 0/0,0 mode: --/PW res: [0x280000403:0x656:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f459a expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:3.0:1775764772.835813:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764772.835815:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764772.835817:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764772.835824:0:2381:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1775764772.835831:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966991232:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764772.836102:0:5561:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1775764772.836115:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978381952 00000100:00100000:1.0:1775764772.836119:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978381952 00000100:00100000:1.0:1775764772.836140:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978382208 00000100:00100000:1.0:1775764772.836143:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978382208 00000100:00100000:1.0:1775764772.836153:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978382336 00000100:00100000:1.0:1775764772.836156:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978382336 00000100:00100000:1.0:1775764772.836164:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978382464 00000100:00100000:1.0:1775764772.836166:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978382464 00000100:00100000:1.0:1775764772.836177:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24178 00000100:00100000:1.0:1775764772.836183:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c8000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:12180:x1862022978381952:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764772.836193:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000095a0503a/0x36f9bdc9e41caa7d lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x52:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ee622 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764772.836238:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764772.836243:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c8000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:12180:x1862022978381952:12345-192.168.202.146@tcp:104: Request processed in 61us (1407us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764772.836249:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24178 00000100:00100000:1.0:1775764772.836262:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24179 00000100:00100000:1.0:1775764772.836268:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c9500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+6:12196:x1862022978382208:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764772.836276:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000b02e1101/0x36f9bdc9e41ca774 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x4a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eddf5 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 71 lru_type: 1 00010000:00010000:1.0:1775764772.836304:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1775764772.836309:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c9500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+6:12196:x1862022978382208:12345-192.168.202.146@tcp:104: Request processed in 42us (1431us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764772.836315:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24179 00000100:00100000:1.0:1775764772.836326:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24180 00000100:00100000:1.0:1775764772.836333:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0cb800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:12039:x1862022978382336:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764772.836340:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000066a2ef3d/0x36f9bdc9e421c1db lrc: 2/0,0 mode: PR/PR res: [0x240000403:0x34:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5f898d expref: -99 pid: 19850 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764772.836689:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:1.0:1775764772.836697:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0cb800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:12039:x1862022978382336:12345-192.168.202.146@tcp:104: Request processed in 363us (1795us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764772.836704:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24180 00000100:00100000:1.0:1775764772.836718:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24181 00000100:00100000:1.0:1775764772.836725:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9966bf5050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:16287:x1862022978382464:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764772.836735:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f2afaa21/0x36f9bdc9e421b17a lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x65b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f5839 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 0 00010000:00010000:1.0:1775764772.836801:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/4 locks to priority blp list, 4 blwis in pool 00000100:00100000:1.0:1775764772.836806:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9966bf5050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:16287:x1862022978382464:12345-192.168.202.146@tcp:104: Request processed in 82us (1883us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764772.836812:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24181 00000100:00100000:0.0:1775764772.836918:0:11146:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 0 new 8192-byte reqbufs (1/1 left), rc = 0 00010000:00010000:3.0:1775764772.837690:0:11138:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 3 total in blp. (3 blwis in pool) 00010000:00010000:3.0:1775764772.837699:0:11138:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000095a0503a/0x36f9bdc9e41caa7d lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x52:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ee622 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:3.0:1775764772.837709:0:11138:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f996695de00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:3.0:1775764772.837714:0:11138:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000095a0503a/0x36f9bdc9e41caa7d lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x52:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ee622 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:3.0:1775764772.837863:0:11138:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000095a0503a/0x36f9bdc9e41caa7d lrc: 2/0,0 mode: --/PW res: [0x280000403:0x52:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee622 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:3.0:1775764772.837871:0:11138:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764772.837888:0:11138:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000095a0503a/0x36f9bdc9e41caa7d lrc: 1/0,0 mode: --/PW res: [0x280000403:0x52:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee622 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:3.0:1775764772.837897:0:11138:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000095a0503a/0x36f9bdc9e41caa7d lrc: 0/0,0 mode: --/PW res: [0x280000403:0x52:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee622 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:3.0:1775764772.837916:0:11138:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:3.0:1775764772.837919:0:11138:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000b02e1101/0x36f9bdc9e41ca774 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x4a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eddf5 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 71 lru_type: 1 00010000:00010000:3.0:1775764772.837937:0:11138:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f996695dc00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:3.0:1775764772.837940:0:11138:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000b02e1101/0x36f9bdc9e41ca774 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x4a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4eddf5 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 71 lru_type: 1 00010000:00010000:3.0:1775764772.837975:0:11138:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000b02e1101/0x36f9bdc9e41ca774 lrc: 2/0,0 mode: --/PW res: [0x240000403:0x4a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eddf5 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 71 lru_type: 1 00010000:00010000:3.0:1775764772.837987:0:11138:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764772.837994:0:11138:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000b02e1101/0x36f9bdc9e41ca774 lrc: 1/0,0 mode: --/PW res: [0x240000403:0x4a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eddf5 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 71 lru_type: 1 00010000:00010000:3.0:1775764772.838001:0:11138:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000b02e1101/0x36f9bdc9e41ca774 lrc: 0/0,0 mode: --/PW res: [0x240000403:0x4a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eddf5 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 71 lru_type: 1 00010000:00010000:3.0:1775764772.838015:0:11138:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1775764772.838018:0:11138:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000066a2ef3d/0x36f9bdc9e421c1db lrc: 2/0,0 mode: PR/PR res: [0x240000403:0x34:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5f898d expref: -99 pid: 19850 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764772.838024:0:11138:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99668b9800 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:3.0:1775764772.838027:0:11138:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000066a2ef3d/0x36f9bdc9e421c1db lrc: 3/0,0 mode: PR/PR res: [0x240000403:0x34:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x428400010000 nid: local remote: 0x2aac6a50e5f898d expref: -99 pid: 19850 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764772.838197:0:11138:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000066a2ef3d/0x36f9bdc9e421c1db lrc: 2/0,0 mode: --/PR res: [0x240000403:0x34:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5f898d expref: -99 pid: 19850 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764772.838205:0:11138:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764772.838209:0:11138:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000066a2ef3d/0x36f9bdc9e421c1db lrc: 1/0,0 mode: --/PR res: [0x240000403:0x34:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5f898d expref: -99 pid: 19850 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764772.838215:0:11138:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000066a2ef3d/0x36f9bdc9e421c1db lrc: 0/0,0 mode: --/PR res: [0x240000403:0x34:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5f898d expref: -99 pid: 19850 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764772.838229:0:11138:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1775764772.838232:0:11138:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f2afaa21/0x36f9bdc9e421b17a lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x65b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f5839 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 0 00010000:00010000:3.0:1775764772.838239:0:11138:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970972200 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:3.0:1775764772.838241:0:11138:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f2afaa21/0x36f9bdc9e421b17a lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x65b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e5f5839 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 0 00010000:00010000:3.0:1775764772.838271:0:11138:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f2afaa21/0x36f9bdc9e421b17a lrc: 2/0,0 mode: --/PW res: [0x240000403:0x65b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f5839 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 0 00010000:00010000:3.0:1775764772.838277:0:11138:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764772.838281:0:11138:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f2afaa21/0x36f9bdc9e421b17a lrc: 1/0,0 mode: --/PW res: [0x240000403:0x65b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f5839 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 0 00010000:00010000:3.0:1775764772.838287:0:11138:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f2afaa21/0x36f9bdc9e421b17a lrc: 0/0,0 mode: --/PW res: [0x240000403:0x65b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f5839 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 0 00010000:00010000:3.0:1775764772.838298:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764772.838299:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764772.838301:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764772.838312:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764772.838339:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966991360:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.838762:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966991616:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.838796:0:2381:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1775764772.838802:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966991488:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.838820:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966991744:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.839677:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c9180 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966990720:192.168.202.146@tcp:101:rm.0 00010000:00010000:3.0:1775764772.839694:0:19896:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c6dc lrc: 4/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fac39 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.839708:0:19896:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c6dc lrc: 4/1,0 mode: --/PR res: [0x200000401:0x277:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fac39 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.839716:0:19896:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c6dc lrc: 4/1,0 mode: --/PR res: [0x200000401:0x277:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fac39 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.839727:0:19896:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c6dc lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x277:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fac39 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.839767:0:19896:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c6dc lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x277:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fac39 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.839779:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c6dc lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x277:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fac39 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.839786:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c6dc lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x277:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fac39 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.839794:0:19896:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c6dc lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x277:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fac39 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764772.839804:0:19896:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994f0c9180 x1862022966990720/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/584 e 0 to 0 dl 1775764788 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1775764772.839820:0:19896:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c6dc lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x277:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fac39 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.839830:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/631 (0 0) 00000080:00200000:3.0:1775764772.839836:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26c:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1775764772.839858:0:19896:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x277:0x0], updating i_size 7168 00000080:00010000:3.0:1775764772.839863:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x277:0x0](ffff8f9952e64b08) for lock 0x36f9bdc9e421c6dc 00000080:00010000:3.0:1775764772.839872:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x277:0x0](ffff8f9952e64b08) 00000080:00010000:3.0:1775764772.839876:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c6dc from it ffffafbac1ec7cb8 00010000:00010000:3.0:1775764772.839879:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c6dc lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x277:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fac39 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764772.839887:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c6dc lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x277:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fac39 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764772.839973:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764772.839981:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=WORD, flags=258 00000080:00200000:3.0:1775764772.839987:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x277:0x0](ffff8f9952e64b08), inode mode 41c0 mask 44 00000080:00200008:3.0:1775764772.839995:0:19896:0:(file.c:1057:ll_file_open()) START file WORD:[0x200000401:0x277:0x0](ffff8f9952e64b08), flags 704400 00000080:00200008:3.0:1775764772.840015:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file WORD:[0x200000401:0x277:0x0](ffff8f9952e64b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764772.840020:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x277:0x0](ffff8f9952e64b08),name=WORD 00000002:00010000:3.0:1775764772.840038:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x277:0x0]) in obj [0x200000401:0x26c:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764772.840185:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.840194:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764772.840225:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966990848:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.840235:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022966990976:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1775764772.840891:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c6dc lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x277:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fac39 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764772.840905:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c6dc lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x277:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fac39 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764772.840918:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26c:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764772.841040:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x277:0x0](ffff8f9952e64b08) 00000080:00010000:3.0:1775764772.841120:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x277:0x0](ffff8f9952e64b08) for lock 0x36f9bdc9e421c6dc 00000080:00010000:3.0:1775764772.841127:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c6dc from it ffffafbac1ec7cd8 00010000:00010000:3.0:1775764772.841131:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c6dc lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x277:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fac39 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764772.841140:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c6dc lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x277:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fac39 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764772.841169:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x277:0x0](ffff8f9952e64b08) pos/size0/7168 32bit_api 0 00000002:00010000:3.0:1775764772.841183:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x277:0x0]) in obj [0x200000401:0x277:0x0], intent: readdir flags 0 00010000:00010000:3.0:1775764772.841191:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c6dc lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x277:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fac39 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764772.841200:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c6dc lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x277:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fac39 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000100:00100000:2.0:1775764772.841692:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966991232:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.841704:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966991104:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.844936:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966991872:192.168.202.146@tcp:37:rm.0 00000100:00100000:3.0:1775764772.871373:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413540 going to sleep for 11 seconds 00000100:00100000:0.0:1775764772.877399:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.877444:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.877474:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764772.877493:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:2.0:1775764772.877951:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966991488:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.877971:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966991360:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.877980:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966991616:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764772.877989:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966989440:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764772.879662:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978383232 00000100:00100000:0.0:1775764772.879677:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978383232 00000100:00100000:0.0:1775764772.879722:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978383616 00000100:00100000:0.0:1775764772.879725:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978383616 00000100:00100000:0.0:1775764772.879735:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978383744 00000100:00100000:0.0:1775764772.879776:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978383744 00000100:00100000:0.0:1775764772.879788:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978383872 00000100:00100000:0.0:1775764772.879790:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978383872 00000100:00100000:0.0:1775764772.879808:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24182 00000100:00100000:0.0:1775764772.879817:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c9880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+7:12051:x1862022978383232:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764772.879836:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000054dab604/0x36f9bdc9e41ca678 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x46:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4edb94 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00010000:00010000:0.0:1775764772.879938:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764772.879945:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c9880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+7:12051:x1862022978383232:12345-192.168.202.146@tcp:104: Request processed in 130us (2558us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764772.879953:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24182 00000100:00100000:0.0:1775764772.879970:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24183 00000100:00100000:0.0:1775764772.879976:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0cbb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+6:12193:x1862022978383616:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764772.879985:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000002f218f80/0x36f9bdc9e41ca6be lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x47:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4edc3c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00010000:00010000:0.0:1775764772.880033:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:0.0:1775764772.880038:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0cbb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+6:12193:x1862022978383616:12345-192.168.202.146@tcp:104: Request processed in 63us (2594us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764772.880195:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24183 00000100:00100000:0.0:1775764772.880209:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24184 00000100:00100000:0.0:1775764772.880215:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0cb100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:12187:x1862022978383744:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764772.880224:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000139dd483/0x36f9bdc9e4210797 lrc: 2/0,0 mode: PR/PR res: [0x280000403:0x45:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5d270c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:0.0:1775764772.880257:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:0.0:1775764772.880263:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0cb100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:12187:x1862022978383744:12345-192.168.202.146@tcp:104: Request processed in 49us (2789us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764772.880269:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24184 00000100:00100000:0.0:1775764772.880280:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24185 00000100:00100000:0.0:1775764772.880299:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c9f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:12040:x1862022978383872:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764772.880308:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000031ed7a52/0x36f9bdc9e41ca591 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x43:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ed95d expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 93 lru_type: 1 00010000:00010000:0.0:1775764772.880639:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/4 locks to priority blp list, 4 blwis in pool 00000100:00100000:0.0:1775764772.880644:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c9f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:12040:x1862022978383872:12345-192.168.202.146@tcp:104: Request processed in 346us (3151us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764772.880651:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24185 00010000:00010000:0.0:1775764772.880668:0:11138:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 3 total in blp. (3 blwis in pool) 00010000:00010000:0.0:1775764772.880675:0:11138:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000054dab604/0x36f9bdc9e41ca678 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x46:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4edb94 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00010000:00010000:0.0:1775764772.880684:0:11138:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970fd6600 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:0.0:1775764772.880690:0:11138:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000054dab604/0x36f9bdc9e41ca678 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x46:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4edb94 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00010000:00010000:0.0:1775764772.880798:0:11138:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000054dab604/0x36f9bdc9e41ca678 lrc: 2/0,0 mode: --/PW res: [0x280000403:0x46:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4edb94 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00010000:00010000:0.0:1775764772.880806:0:11138:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764772.880834:0:11138:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000054dab604/0x36f9bdc9e41ca678 lrc: 1/0,0 mode: --/PW res: [0x280000403:0x46:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4edb94 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00010000:00010000:0.0:1775764772.880848:0:11138:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000054dab604/0x36f9bdc9e41ca678 lrc: 0/0,0 mode: --/PW res: [0x280000403:0x46:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4edb94 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00010000:00010000:0.0:1775764772.880872:0:11138:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:0.0:1775764772.880876:0:11138:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000002f218f80/0x36f9bdc9e41ca6be lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x47:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4edc3c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00010000:00010000:0.0:1775764772.880883:0:11138:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970fd7800 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:0.0:1775764772.880887:0:11138:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000002f218f80/0x36f9bdc9e41ca6be lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x47:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4edc3c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00010000:00010000:0.0:1775764772.880919:0:11138:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000002f218f80/0x36f9bdc9e41ca6be lrc: 2/0,0 mode: --/PW res: [0x280000403:0x47:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4edc3c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00010000:00010000:0.0:1775764772.880937:0:11138:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764772.880944:0:11138:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000002f218f80/0x36f9bdc9e41ca6be lrc: 1/0,0 mode: --/PW res: [0x280000403:0x47:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4edc3c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00010000:00010000:0.0:1775764772.880951:0:11138:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000002f218f80/0x36f9bdc9e41ca6be lrc: 0/0,0 mode: --/PW res: [0x280000403:0x47:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4edc3c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00010000:00010000:0.0:1775764772.880965:0:11138:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1775764772.880969:0:11138:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000139dd483/0x36f9bdc9e4210797 lrc: 2/0,0 mode: PR/PR res: [0x280000403:0x45:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5d270c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:0.0:1775764772.880982:0:11138:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f994ed0c000 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:0.0:1775764772.880986:0:11138:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000139dd483/0x36f9bdc9e4210797 lrc: 3/0,0 mode: PR/PR res: [0x280000403:0x45:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x428400010000 nid: local remote: 0x2aac6a50e5d270c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:0.0:1775764772.881016:0:11138:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000139dd483/0x36f9bdc9e4210797 lrc: 2/0,0 mode: --/PR res: [0x280000403:0x45:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5d270c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:0.0:1775764772.881023:0:11138:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764772.881028:0:11138:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000139dd483/0x36f9bdc9e4210797 lrc: 1/0,0 mode: --/PR res: [0x280000403:0x45:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5d270c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:0.0:1775764772.881035:0:11138:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000139dd483/0x36f9bdc9e4210797 lrc: 0/0,0 mode: --/PR res: [0x280000403:0x45:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5d270c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:0.0:1775764772.882811:0:11138:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1775764772.882821:0:11138:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000031ed7a52/0x36f9bdc9e41ca591 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x43:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ed95d expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 93 lru_type: 1 00010000:00010000:0.0:1775764772.882834:0:11138:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970fd7600 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:0.0:1775764772.882840:0:11138:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000031ed7a52/0x36f9bdc9e41ca591 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x43:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ed95d expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 93 lru_type: 1 00010000:00010000:0.0:1775764772.882944:0:11138:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000031ed7a52/0x36f9bdc9e41ca591 lrc: 2/0,0 mode: --/PW res: [0x280000403:0x43:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed95d expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 93 lru_type: 1 00010000:00010000:0.0:1775764772.882958:0:11138:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764772.882968:0:11138:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000031ed7a52/0x36f9bdc9e41ca591 lrc: 1/0,0 mode: --/PW res: [0x280000403:0x43:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed95d expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 93 lru_type: 1 00010000:00010000:0.0:1775764772.882975:0:11138:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000031ed7a52/0x36f9bdc9e41ca591 lrc: 0/0,0 mode: --/PW res: [0x280000403:0x43:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ed95d expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 93 lru_type: 1 00010000:00010000:0.0:1775764772.882990:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764772.882992:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764772.882995:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764772.883011:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764772.883018:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764772.883032:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764772.883233:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022966991744:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764772.884170:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966992000:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764772.884258:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966992256:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764772.884723:0:2380:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:0.0:1775764772.884735:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966992128:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764772.884857:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966992384:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764772.918554:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966992128:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764772.918573:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0caa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966992384:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764772.918582:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0cb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966992000:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764772.918591:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022966992256:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764772.952961:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966991872:192.168.202.146@tcp:37:rm.0 00010000:00010000:3.0:1775764772.954503:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c6dc lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x277:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fac39 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764772.954523:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c6dc lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x277:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fac39 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764772.955517:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x277:0x0](ffff8f9952e64b08) pos/size18446744073709551614/7168 32bit_api 0 00000080:00200008:2.0:1775764772.955540:0:19896:0:(file.c:447:ll_file_release()) START file WORD:[0x200000401:0x277:0x0](ffff8f9952e64b08), flags 704000 00010000:00010000:2.0:1775764772.955569:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3b8c00/0x36f9bdc9e41ca169 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x277:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ecdfd expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764772.955616:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:2.0:1775764772.955626:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=WORD, flags=0 00000080:00200000:2.0:1775764772.955631:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 3 00000080:00200000:2.0:1775764772.955637:0:19896:0:(namei.c:2305:ll_rmdir()) VFS Op:name=WORD, dir=[0x200000401:0x26c:0x0](ffff8f9952db6c08) 00010000:00010000:2.0:1775764772.955665:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c6dc lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x277:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fac39 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764772.955864:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3b8000/0x36f9bdc9e41ca1a8 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x277:0x0].0x0 bits 0x11/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e4ecec1 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 211 lru_type: 1 00010000:00010000:2.0:1775764772.955879:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3b8c00/0x36f9bdc9e41ca169 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x277:0x0].0x0 bits 0x5/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e4ecdfd expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000002:00100000:2.0:1775764772.956628:0:19896:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f99463c3b80 x1862022950269312/t4294969493(4294969493) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764067 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764772.956641:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764772.956647:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994efdce00 x1862022966992640/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764772.956672:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdce00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966992640:192.168.202.146@tcp:35:rm.0 00000100:00100000:2.0:1775764772.956858:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102480 going to sleep for 11 seconds 00000100:00100000:2.0:1775764772.975341:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdce00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966992640:192.168.202.146@tcp:35:rm.0 00000002:00100000:2.0:1775764772.975380:0:19896:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f99463c3b80 x1862022950269312/t4294969493(4294969493) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764067 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00010000:00010000:2.0:1775764772.975501:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c6dc lrc: 1/0,0 mode: --/PR res: [0x200000401:0x277:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fac39 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764772.975512:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3b8000/0x36f9bdc9e41ca1a8 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x277:0x0].0x0 bits 0x11/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ecec1 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 211 lru_type: 1 00010000:00010000:2.0:1775764772.975519:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3b8c00/0x36f9bdc9e41ca169 lrc: 1/0,0 mode: --/CR res: [0x200000401:0x277:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ecdfd expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764772.975530:0:19896:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:2.0:1775764772.975533:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c6dc lrc: 0/0,0 mode: --/PR res: [0x200000401:0x277:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fac39 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764772.975545:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3b8000/0x36f9bdc9e41ca1a8 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x277:0x0].0x0 bits 0x11/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ecec1 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 211 lru_type: 1 00010000:00010000:2.0:1775764772.975552:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3b8c00/0x36f9bdc9e41ca169 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x277:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ecdfd expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000020:00100000:2.0:1775764772.975572:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1775764772.975578:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994efdce00 x1862022966992896/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764772.975611:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdce00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966992896:192.168.202.146@tcp:36:rm.0 00000100:00100000:2.0:1775764772.975693:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102480 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.002684:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdce00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966992896:192.168.202.146@tcp:36:rm.0 00000080:00200000:2.0:1775764773.002728:0:19896:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x277:0x0](ffff8f9952e64b08) 00000080:00200000:2.0:1775764773.002890:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:2.0:1775764773.002909:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PWRPNT, flags=0 00000080:00200008:2.0:1775764773.002919:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file PWRPNT:[0x200000401:0x285:0x0](ffff8f9952da3a88), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764773.002924:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x285:0x0](ffff8f9952da3a88),name=PWRPNT 00000002:00010000:2.0:1775764773.002941:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x285:0x0]) in obj [0x200000401:0x26c:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764773.002953:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/645 (0 0) 00010000:00010000:2.0:1775764773.002995:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c6e3 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.003008:0:19896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c6e3 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.003018:0:19896:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c6e3 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764773.003036:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdf480 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966993024:192.168.202.146@tcp:101:rm.0 00000100:00100000:2.0:1775764773.003119:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102480 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.020830:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdf480 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966993024:192.168.202.146@tcp:101:rm.0 00010000:00010000:2.0:1775764773.020856:0:19896:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c6e3 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fac78 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.020874:0:19896:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c6e3 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x285:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fac78 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.020882:0:19896:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c6e3 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fac78 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.020894:0:19896:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c6e3 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fac78 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.020919:0:19896:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c6e3 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fac78 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.020941:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c6e3 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fac78 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.020955:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c6e3 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fac78 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.020963:0:19896:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c6e3 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fac78 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764773.020976:0:19896:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994efdf480 x1862022966993024/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/584 e 0 to 0 dl 1775764789 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764773.020988:0:19896:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c6e3 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fac78 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.020998:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/645 (0 0) 00000080:00200000:2.0:1775764773.021005:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26c:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764773.021027:0:19896:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x285:0x0], updating i_size 7168 00000080:00010000:2.0:1775764773.021033:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x285:0x0](ffff8f9952da3a88) for lock 0x36f9bdc9e421c6e3 00000080:00010000:2.0:1775764773.021053:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x285:0x0](ffff8f9952da3a88) 00000080:00010000:2.0:1775764773.021058:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c6e3 from it ffffafbac1ec7cb8 00010000:00010000:2.0:1775764773.021061:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c6e3 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fac78 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.021071:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c6e3 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fac78 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764773.021149:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:2.0:1775764773.021159:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PWRPNT, flags=258 00000080:00200000:2.0:1775764773.021164:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x285:0x0](ffff8f9952da3a88), inode mode 41c0 mask 44 00000080:00200008:2.0:1775764773.021174:0:19896:0:(file.c:1057:ll_file_open()) START file PWRPNT:[0x200000401:0x285:0x0](ffff8f9952da3a88), flags 704400 00000080:00200008:2.0:1775764773.021192:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file PWRPNT:[0x200000401:0x285:0x0](ffff8f9952da3a88), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764773.021198:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x285:0x0](ffff8f9952da3a88),name=PWRPNT 00000002:00010000:2.0:1775764773.021217:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x285:0x0]) in obj [0x200000401:0x26c:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764773.021227:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c6e3 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fac78 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764773.021238:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c6e3 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fac78 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764773.021245:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26c:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764773.021250:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x285:0x0](ffff8f9952da3a88) 00000080:00010000:2.0:1775764773.021253:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x285:0x0](ffff8f9952da3a88) for lock 0x36f9bdc9e421c6e3 00000080:00010000:2.0:1775764773.021257:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c6e3 from it ffffafbac1ec7cd8 00010000:00010000:2.0:1775764773.021260:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c6e3 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fac78 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764773.021268:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c6e3 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fac78 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764773.021291:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x285:0x0](ffff8f9952da3a88) pos/size0/7168 32bit_api 0 00000002:00010000:2.0:1775764773.021304:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x285:0x0]) in obj [0x200000401:0x285:0x0], intent: readdir flags 0 00010000:00010000:2.0:1775764773.021310:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c6e3 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fac78 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764773.021611:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c6e3 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fac78 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000100:00100000:2.0:1775764773.022572:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdf480 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966993152:192.168.202.146@tcp:37:rm.0 00000100:00100000:2.0:1775764773.022679:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102480 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.034296:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdf480 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966993152:192.168.202.146@tcp:37:rm.0 00010000:00010000:2.0:1775764773.037736:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c6e3 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fac78 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764773.037786:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c6e3 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fac78 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764773.037838:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x285:0x0](ffff8f9952da3a88) pos/size18446744073709551614/7168 32bit_api 0 00000080:00200008:2.0:1775764773.037857:0:19896:0:(file.c:447:ll_file_release()) START file PWRPNT:[0x200000401:0x285:0x0](ffff8f9952da3a88), flags 704000 00010000:00010000:2.0:1775764773.037884:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971226e00/0x36f9bdc9e41ca38b lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x285:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ed3d7 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 68 lru_type: 1 00000080:00200000:2.0:1775764773.037920:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:2.0:1775764773.037931:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PWRPNT, flags=0 00000080:00200000:2.0:1775764773.037936:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 3 00000080:00200000:2.0:1775764773.037942:0:19896:0:(namei.c:2305:ll_rmdir()) VFS Op:name=PWRPNT, dir=[0x200000401:0x26c:0x0](ffff8f9952db6c08) 00010000:00010000:2.0:1775764773.037968:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c6e3 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fac78 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764773.038037:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971226c00/0x36f9bdc9e41ca3bc lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x285:0x0].0x0 bits 0x11/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e4ed47f expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 70 lru_type: 1 00010000:00010000:2.0:1775764773.038051:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971226e00/0x36f9bdc9e41ca38b lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x285:0x0].0x0 bits 0x5/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e4ed3d7 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 68 lru_type: 1 00000002:00100000:2.0:1775764773.038137:0:19896:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f994f0ab100 x1862022950290688/t4294969536(4294969536) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764068 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764773.038150:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764773.038156:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994efdf480 x1862022966993408/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764773.038182:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdf480 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966993408:192.168.202.146@tcp:35:rm.0 00000100:00100000:2.0:1775764773.038570:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102480 going to sleep for 11 seconds 00000100:00100000:3.0:1775764773.055254:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdf480 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966993408:192.168.202.146@tcp:35:rm.0 00000002:00100000:3.0:1775764773.055309:0:19896:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f994f0ab100 x1862022950290688/t4294969536(4294969536) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764068 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00010000:00010000:3.0:1775764773.055472:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c6e3 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fac78 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764773.055482:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971226c00/0x36f9bdc9e41ca3bc lrc: 1/0,0 mode: --/PR res: [0x200000401:0x285:0x0].0x0 bits 0x11/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ed47f expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 70 lru_type: 1 00010000:00010000:3.0:1775764773.055489:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971226e00/0x36f9bdc9e41ca38b lrc: 1/0,0 mode: --/CR res: [0x200000401:0x285:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ed3d7 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 68 lru_type: 1 00010000:00010000:3.0:1775764773.055496:0:19896:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:3.0:1775764773.055499:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c6e3 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x285:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fac78 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764773.055511:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971226c00/0x36f9bdc9e41ca3bc lrc: 0/0,0 mode: --/PR res: [0x200000401:0x285:0x0].0x0 bits 0x11/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ed47f expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 70 lru_type: 1 00010000:00010000:3.0:1775764773.055519:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971226e00/0x36f9bdc9e41ca38b lrc: 0/0,0 mode: --/CR res: [0x200000401:0x285:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ed3d7 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 68 lru_type: 1 00000020:00100000:3.0:1775764773.055541:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1775764773.055548:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970eddf80 x1862022966993664/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764773.055572:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966993664:192.168.202.146@tcp:36:rm.0 00000100:00100000:3.0:1775764773.055675:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413540 going to sleep for 11 seconds 00000100:00100000:3.0:1775764773.067376:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966993664:192.168.202.146@tcp:36:rm.0 00000080:00200000:3.0:1775764773.067419:0:19896:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x285:0x0](ffff8f9952da3a88) 00000080:00200000:3.0:1775764773.067522:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764773.067532:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=SEED, flags=0 00000080:00200008:3.0:1775764773.067543:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file SEED:[0x200000401:0x2ea:0x0](ffff8f9952dd7448), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764773.067549:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2ea:0x0](ffff8f9952dd7448),name=SEED 00000002:00010000:3.0:1775764773.067571:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2ea:0x0]) in obj [0x200000401:0x26c:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764773.067586:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/746 (0 0) 00010000:00010000:3.0:1775764773.067652:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6ea lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.067662:0:19896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6ea lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.067672:0:19896:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6ea lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764773.067696:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966993792:192.168.202.146@tcp:101:rm.0 00000100:00100000:3.0:1775764773.067834:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413540 going to sleep for 11 seconds 00000100:00100000:3.0:1775764773.074118:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966993792:192.168.202.146@tcp:101:rm.0 00010000:00010000:3.0:1775764773.074144:0:19896:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6ea lrc: 4/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fac9b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.074160:0:19896:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6ea lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2ea:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fac9b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.074168:0:19896:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6ea lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2ea:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fac9b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.074181:0:19896:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6ea lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2ea:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fac9b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.074191:0:19896:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6ea lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2ea:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fac9b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.074204:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6ea lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2ea:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fac9b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.074212:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6ea lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2ea:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fac9b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.074219:0:19896:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6ea lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2ea:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fac9b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764773.074231:0:19896:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970edf800 x1862022966993792/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/584 e 0 to 0 dl 1775764789 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1775764773.074243:0:19896:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6ea lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2ea:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fac9b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.074253:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/746 (0 0) 00000080:00200000:3.0:1775764773.074260:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26c:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1775764773.074279:0:19896:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x2ea:0x0], updating i_size 7168 00000080:00010000:3.0:1775764773.074298:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2ea:0x0](ffff8f9952dd7448) for lock 0x36f9bdc9e421c6ea 00000080:00010000:3.0:1775764773.074308:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2ea:0x0](ffff8f9952dd7448) 00000080:00010000:3.0:1775764773.074311:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c6ea from it ffffafbac1ec7cb8 00010000:00010000:3.0:1775764773.074584:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6ea lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2ea:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fac9b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.074593:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6ea lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ea:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fac9b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764773.074673:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764773.074682:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=SEED, flags=258 00000080:00200000:3.0:1775764773.074688:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2ea:0x0](ffff8f9952dd7448), inode mode 41c0 mask 44 00000080:00200008:3.0:1775764773.074695:0:19896:0:(file.c:1057:ll_file_open()) START file SEED:[0x200000401:0x2ea:0x0](ffff8f9952dd7448), flags 704400 00000080:00200008:3.0:1775764773.074713:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file SEED:[0x200000401:0x2ea:0x0](ffff8f9952dd7448), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764773.074719:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2ea:0x0](ffff8f9952dd7448),name=SEED 00000002:00010000:3.0:1775764773.074736:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2ea:0x0]) in obj [0x200000401:0x26c:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764773.075114:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6ea lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2ea:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fac9b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764773.087239:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6ea lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2ea:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fac9b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764773.087262:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26c:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764773.087276:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2ea:0x0](ffff8f9952dd7448) 00000080:00010000:0.0:1775764773.087279:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2ea:0x0](ffff8f9952dd7448) for lock 0x36f9bdc9e421c6ea 00000080:00010000:0.0:1775764773.087290:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c6ea from it ffffafbac1ec7cd8 00010000:00010000:0.0:1775764773.087294:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6ea lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2ea:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fac9b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764773.087305:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6ea lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ea:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fac9b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764773.087352:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2ea:0x0](ffff8f9952dd7448) pos/size0/7168 32bit_api 0 00000002:00010000:0.0:1775764773.087388:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2ea:0x0]) in obj [0x200000401:0x2ea:0x0], intent: readdir flags 0 00010000:00010000:0.0:1775764773.087402:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6ea lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2ea:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fac9b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764773.087411:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6ea lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2ea:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fac9b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000100:00100000:0.0:1775764773.088277:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c9c00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966993920:192.168.202.146@tcp:37:rm.0 00000100:00100000:0.0:1775764773.088383:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f6c0 going to sleep for 11 seconds 00000100:00100000:0.0:1775764773.108703:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c9c00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966993920:192.168.202.146@tcp:37:rm.0 00010000:00010000:0.0:1775764773.109512:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6ea lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2ea:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fac9b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764773.109529:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6ea lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ea:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fac9b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1775764773.109579:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2ea:0x0](ffff8f9952dd7448) pos/size18446744073709551614/7168 32bit_api 0 00000080:00200008:0.0:1775764773.109607:0:19896:0:(file.c:447:ll_file_release()) START file SEED:[0x200000401:0x2ea:0x0](ffff8f9952dd7448), flags 704000 00010000:00010000:0.0:1775764773.109632:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4400/0x36f9bdc9e41cb1b5 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2ea:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ef87b expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:0.0:1775764773.109667:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764773.109677:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=SEED, flags=0 00000080:00200000:0.0:1775764773.109683:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 3 00000080:00200000:0.0:1775764773.109688:0:19896:0:(namei.c:2305:ll_rmdir()) VFS Op:name=SEED, dir=[0x200000401:0x26c:0x0](ffff8f9952db6c08) 00010000:00010000:0.0:1775764773.109712:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6ea lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ea:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fac9b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764773.110030:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c4c1 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2ea:0x0].0x0 bits 0x11/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f9ce9 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764773.110121:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4400/0x36f9bdc9e41cb1b5 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2ea:0x0].0x0 bits 0x5/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e4ef87b expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000002:00100000:0.0:1775764773.110175:0:19896:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f9970f68380 x1862022950432896/t4294969772(4294969772) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764076 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1775764773.110187:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1775764773.110193:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f0c9c00 x1862022966994176/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1775764773.110216:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c9c00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966994176:192.168.202.146@tcp:35:rm.0 00000100:00100000:1.0:1775764773.111422:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f6c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.115291:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c9c00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966994176:192.168.202.146@tcp:35:rm.0 00000002:00100000:2.0:1775764773.115333:0:19896:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f9970f68380 x1862022950432896/t4294969772(4294969772) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764076 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00010000:00010000:2.0:1775764773.115437:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6ea lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2ea:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fac9b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764773.115446:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c4c1 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2ea:0x0].0x0 bits 0x11/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9ce9 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764773.115453:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4400/0x36f9bdc9e41cb1b5 lrc: 1/0,0 mode: --/CR res: [0x200000401:0x2ea:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ef87b expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1775764773.115460:0:19896:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:2.0:1775764773.115462:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5000/0x36f9bdc9e421c6ea lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2ea:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fac9b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764773.115474:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c4c1 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2ea:0x0].0x0 bits 0x11/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f9ce9 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764773.115497:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4400/0x36f9bdc9e41cb1b5 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x2ea:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ef87b expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000020:00100000:2.0:1775764773.115519:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1775764773.115525:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994efdea00 x1862022966994432/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764773.115549:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdea00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966994432:192.168.202.146@tcp:36:rm.0 00000100:00100000:2.0:1775764773.115639:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102c00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.121019:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdea00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966994432:192.168.202.146@tcp:36:rm.0 00000080:00200000:2.0:1775764773.121075:0:19896:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2ea:0x0](ffff8f9952dd7448) 00000080:00200000:2.0:1775764773.121163:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:2.0:1775764773.121172:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=0 00000080:00200008:2.0:1775764773.121183:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file PARADOX:[0x200000401:0x2bb:0x0](ffff8f9952da4b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764773.121188:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08),name=PARADOX 00000002:00010000:2.0:1775764773.121208:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2bb:0x0]) in obj [0x200000401:0x26c:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764773.121221:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/699 (0 0) 00010000:00010000:2.0:1775764773.121271:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f1 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.121280:0:19896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f1 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.121290:0:19896:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f1 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764773.121310:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efddc00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966994560:192.168.202.146@tcp:101:rm.0 00000100:00100000:2.0:1775764773.121399:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102c00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.125233:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efddc00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966994560:192.168.202.146@tcp:101:rm.0 00010000:00010000:2.0:1775764773.125257:0:19896:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f1 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5facbe expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.125274:0:19896:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f1 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5facbe expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.125293:0:19896:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f1 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5facbe expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.125306:0:19896:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f1 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5facbe expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.125324:0:19896:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f1 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5facbe expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.125338:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f1 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5facbe expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.125346:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f1 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5facbe expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.125352:0:19896:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f1 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5facbe expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764773.125363:0:19896:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994efddc00 x1862022966994560/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/584 e 0 to 0 dl 1775764789 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764773.125376:0:19896:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f1 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5facbe expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.125386:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/699 (0 0) 00000080:00200000:2.0:1775764773.125392:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26c:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764773.125413:0:19896:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x2bb:0x0], updating i_size 7168 00000080:00010000:2.0:1775764773.125419:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2bb:0x0](ffff8f9952da4b08) for lock 0x36f9bdc9e421c6f1 00000080:00010000:2.0:1775764773.125427:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2bb:0x0](ffff8f9952da4b08) 00000080:00010000:2.0:1775764773.125430:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c6f1 from it ffffafbac1ec7cb8 00010000:00010000:2.0:1775764773.125433:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f1 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5facbe expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.125441:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f1 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5facbe expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764773.125520:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:2.0:1775764773.125529:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=258 00000080:00200000:2.0:1775764773.125535:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08), inode mode 41c0 mask 44 00000080:00200008:2.0:1775764773.125543:0:19896:0:(file.c:1057:ll_file_open()) START file PARADOX:[0x200000401:0x2bb:0x0](ffff8f9952da4b08), flags 704400 00000080:00200008:2.0:1775764773.125563:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file PARADOX:[0x200000401:0x2bb:0x0](ffff8f9952da4b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764773.125568:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08),name=PARADOX 00000002:00010000:2.0:1775764773.125584:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2bb:0x0]) in obj [0x200000401:0x26c:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764773.125593:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5facbe expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764773.125617:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5facbe expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764773.125628:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26c:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764773.125631:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2bb:0x0](ffff8f9952da4b08) 00000080:00010000:2.0:1775764773.125634:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2bb:0x0](ffff8f9952da4b08) for lock 0x36f9bdc9e421c6f1 00000080:00010000:2.0:1775764773.125637:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c6f1 from it ffffafbac1ec7cd8 00010000:00010000:2.0:1775764773.125640:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f1 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5facbe expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764773.125646:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f1 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5facbe expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764773.125667:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08) pos/size0/7168 32bit_api 0 00000002:00010000:2.0:1775764773.125680:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2bb:0x0]) in obj [0x200000401:0x2bb:0x0], intent: readdir flags 0 00010000:00010000:2.0:1775764773.125686:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5facbe expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764773.125692:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5facbe expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000100:00100000:2.0:1775764773.128839:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efddc00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966994688:192.168.202.146@tcp:37:rm.0 00000100:00100000:2.0:1775764773.130198:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102c00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.146775:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efddc00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966994688:192.168.202.146@tcp:37:rm.0 00010000:00010000:2.0:1775764773.147479:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f1 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5facbe expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764773.147495:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f1 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5facbe expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764773.147544:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08) pos/size18446744073709551614/7168 32bit_api 0 00000080:00200008:2.0:1775764773.147564:0:19896:0:(file.c:447:ll_file_release()) START file PARADOX:[0x200000401:0x2bb:0x0](ffff8f9952da4b08), flags 704000 00010000:00010000:2.0:1775764773.147589:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d9c00/0x36f9bdc9e41cab4f lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ee860 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764773.147634:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 1 00000080:00200000:2.0:1775764773.147644:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=0 00000080:00200000:2.0:1775764773.147650:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08), inode mode 41c0 mask 3 00000080:00200000:2.0:1775764773.147655:0:19896:0:(namei.c:2305:ll_rmdir()) VFS Op:name=PARADOX, dir=[0x200000401:0x26c:0x0](ffff8f9952db6c08) 00010000:00010000:2.0:1775764773.147684:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f1 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5facbe expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764773.147811:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970adec00/0x36f9bdc9e41cab8e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x11/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e4ee908 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764773.147827:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d9c00/0x36f9bdc9e41cab4f lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x5/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e4ee860 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000002:00100000:2.0:1775764773.147872:0:19896:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f9970f14e00 x1862022950369024/t4294969666(4294969666) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764072 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764773.147883:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764773.147889:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994efddc00 x1862022966994944/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764773.147912:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efddc00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966994944:192.168.202.146@tcp:35:rm.0 00000100:00100000:2.0:1775764773.148007:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102c00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.158570:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efddc00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966994944:192.168.202.146@tcp:35:rm.0 00000002:00100000:2.0:1775764773.158614:0:19896:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f9970f14e00 x1862022950369024/t4294969666(4294969666) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764072 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00010000:00010000:2.0:1775764773.158710:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f1 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5facbe expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764773.158721:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970adec00/0x36f9bdc9e41cab8e lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x11/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ee908 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764773.158727:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d9c00/0x36f9bdc9e41cab4f lrc: 1/0,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ee860 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764773.158732:0:19896:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:2.0:1775764773.158735:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f1 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5facbe expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764773.158786:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970adec00/0x36f9bdc9e41cab8e lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2bb:0x0].0x0 bits 0x11/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ee908 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764773.158794:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d9c00/0x36f9bdc9e41cab4f lrc: 0/0,0 mode: --/CR res: [0x200000401:0x2bb:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ee860 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000020:00100000:2.0:1775764773.158814:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1775764773.158820:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994efddc00 x1862022966995200/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764773.158841:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efddc00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966995200:192.168.202.146@tcp:36:rm.0 00000100:00100000:2.0:1775764773.158983:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102c00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.179586:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efddc00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966995200:192.168.202.146@tcp:36:rm.0 00000080:00200000:2.0:1775764773.179651:0:19896:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2bb:0x0](ffff8f9952da4b08) 00000080:00200008:2.0:1775764773.179795:0:19896:0:(file.c:447:ll_file_release()) START file ~dmtmp:[0x200000401:0x26c:0x0](ffff8f9952db6c08), flags 704000 00010000:00010000:2.0:1775764773.179823:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971078400/0x36f9bdc9e41ca50c lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ed7b9 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764773.179860:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 1 00000080:00200000:2.0:1775764773.179875:0:19896:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=~dmtmp, dir=[0x200000401:0x26a:0x0](ffff8f9952da00c8), flags=0 00000080:00200000:2.0:1775764773.179880:0:19896:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=~dmtmp, dir=[0x200000401:0x26a:0x0](ffff8f9952da00c8), intent=getattr 00000002:00010000:2.0:1775764773.179902:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ~dmtmp,[0x0:0x0:0x0]) in obj [0x200000401:0x26a:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764773.180154:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f8 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26a:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.180163:0:19896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f8 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26a:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.180172:0:19896:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f8 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26a:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764773.180194:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efde300 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966995328:192.168.202.146@tcp:101:rm.0 00000100:00100000:2.0:1775764773.180276:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102c00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.184411:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efde300 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966995328:192.168.202.146@tcp:101:rm.0 00010000:00010000:2.0:1775764773.184441:0:19896:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f8 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x26a:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5facef expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.184459:0:19896:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f8 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5facef expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.184471:0:19896:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f8 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5facef expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.184481:0:19896:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f8 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5facef expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.184492:0:19896:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f8 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5facef expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.184504:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f8 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5facef expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.184513:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f8 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5facef expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.184520:0:19896:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f8 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5facef expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764773.184531:0:19896:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994efde300 x1862022966995328/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/584 e 0 to 0 dl 1775764789 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764773.184543:0:19896:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f8 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5facef expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.184552:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/620 (0 0) 00000080:00200000:2.0:1775764773.184559:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26a:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764773.184583:0:19896:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x26c:0x0], updating i_size 7168 00000080:00200000:2.0:1775764773.184586:0:19896:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x26c:0x0](ffff8f9952db6c08): rc = 0 00000080:00010000:2.0:1775764773.184593:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x26c:0x0](ffff8f9952db6c08) for lock 0x36f9bdc9e421c6f8 00000080:00010000:2.0:1775764773.184615:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x26c:0x0](ffff8f9952db6c08) 00000080:00010000:2.0:1775764773.184620:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c6f8 from it ffffafbac1ec7e00 00010000:00010000:2.0:1775764773.184623:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f8 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5facef expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.184631:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f8 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5facef expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764773.184675:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8), inode mode 41c0 mask 3 00000080:00200000:2.0:1775764773.184686:0:19896:0:(namei.c:2305:ll_rmdir()) VFS Op:name=~dmtmp, dir=[0x200000401:0x26a:0x0](ffff8f9952da00c8) 00010000:00010000:2.0:1775764773.184712:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c6a4 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26a:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fa8ea expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764773.184732:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b9000/0x36f9bdc9e421c228 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26a:0x0].0x0 bits 0x11/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8aac expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764773.184834:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f8 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5facef expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764773.184848:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971078400/0x36f9bdc9e41ca50c lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e4ed7b9 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000002:00100000:2.0:1775764773.184890:0:19896:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f9970c00e00 x1862022950308480/t4294969562(4294969562) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764069 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764773.184901:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764773.184907:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994efde300 x1862022966995584/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764773.184932:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efde300 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966995584:192.168.202.146@tcp:35:rm.0 00000100:00100000:2.0:1775764773.185009:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102c00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.192347:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efde300 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966995584:192.168.202.146@tcp:35:rm.0 00000002:00100000:2.0:1775764773.192380:0:19896:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f9970c00e00 x1862022950308480/t4294969562(4294969562) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764069 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00010000:00010000:2.0:1775764773.192459:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f8 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5facef expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764773.192469:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971078400/0x36f9bdc9e41ca50c lrc: 1/0,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ed7b9 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764773.192475:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c6a4 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x26a:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fa8ea expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764773.192481:0:19896:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:2.0:1775764773.192484:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c6f8 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x26c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5facef expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764773.192496:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971078400/0x36f9bdc9e41ca50c lrc: 0/0,0 mode: --/CR res: [0x200000401:0x26c:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ed7b9 expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764773.192511:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c6a4 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x26a:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fa8ea expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000020:00100000:2.0:1775764773.192526:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1775764773.192532:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994efde300 x1862022966995840/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764773.192556:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efde300 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966995840:192.168.202.146@tcp:36:rm.0 00000100:00100000:2.0:1775764773.192658:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102c00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.197568:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efde300 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966995840:192.168.202.146@tcp:36:rm.0 00000080:00200000:2.0:1775764773.197622:0:19896:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x26c:0x0](ffff8f9952db6c08) 00000080:00200008:2.0:1775764773.197705:0:19896:0:(file.c:447:ll_file_release()) START file client0:[0x200000401:0x26a:0x0](ffff8f9952da00c8), flags 704000 00010000:00010000:2.0:1775764773.197731:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c6b2 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x26a:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa906 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764773.197785:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:2.0:1775764773.197795:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client0, flags=0 00000080:00200000:2.0:1775764773.197801:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 3 00000080:00200000:2.0:1775764773.197806:0:19896:0:(namei.c:2305:ll_rmdir()) VFS Op:name=client0, dir=[0x200000401:0x268:0x0](ffff8f9952e63a88) 00010000:00010000:2.0:1775764773.197832:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd800/0x36f9bdc9e421c68f lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x268:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fa8b9 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764773.197848:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb400/0x36f9bdc9e421c69d lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x268:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa8d5 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764773.197920:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c6b2 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x26a:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fa906 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000002:00100000:2.0:1775764773.197965:0:19896:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f9970680000 x1862022966976256/t4294997534(4294997534) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764788 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'rm.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764773.197977:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764773.197983:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994efdf800 x1862022966996096/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764773.198011:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdf800 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966996096:192.168.202.146@tcp:35:rm.0 00000100:00100000:3.0:1775764773.202242:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102c00 going to sleep for 11 seconds 00000100:00100000:3.0:1775764773.206102:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdf800 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966996096:192.168.202.146@tcp:35:rm.0 00000002:00100000:3.0:1775764773.206142:0:19896:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f9970680000 x1862022966976256/t4294997534(4294997534) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764788 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'rm.0' uid:0 gid:0 projid:0 00010000:00010000:3.0:1775764773.206202:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b9000/0x36f9bdc9e421c228 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26a:0x0].0x0 bits 0x11/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f8aac expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764773.206248:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c6b2 lrc: 1/0,0 mode: --/CR res: [0x200000401:0x26a:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fa906 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764773.206256:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b9000/0x36f9bdc9e421c228 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x26a:0x0].0x0 bits 0x11/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8aac expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764773.206264:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd800/0x36f9bdc9e421c68f lrc: 1/0,0 mode: --/PR res: [0x200000401:0x268:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fa8b9 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1775764773.206269:0:19896:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:3.0:1775764773.206272:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c6b2 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x26a:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fa906 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764773.206284:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b9000/0x36f9bdc9e421c228 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x26a:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8aac expref: -99 pid: 19723 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764773.206297:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd800/0x36f9bdc9e421c68f lrc: 0/0,0 mode: --/PR res: [0x200000401:0x268:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fa8b9 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000020:00100000:3.0:1775764773.206312:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1775764773.206318:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970edf800 x1862022966996352/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764773.206339:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966996352:192.168.202.146@tcp:36:rm.0 00000100:00100000:3.0:1775764773.206688:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413540 going to sleep for 11 seconds 00000100:00100000:3.0:1775764773.218141:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966996352:192.168.202.146@tcp:36:rm.0 00000080:00200000:0.0:1775764773.218606:0:19896:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x26a:0x0](ffff8f9952da00c8) 00000080:00200000:0.0:1775764773.218868:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764773.218879:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=0 00000080:00200008:0.0:1775764773.218891:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file client1:[0x200000401:0x269:0x0](ffff8f9952db63c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764773.218896:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8),name=client1 00000002:00010000:0.0:1775764773.218917:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x269:0x0]) in obj [0x200000401:0x268:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764773.218942:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/617 (0 0) 00010000:00010000:0.0:1775764773.218997:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c6ff lrc: 3/1,0 mode: --/CR res: [0x200000401:0x268:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.219007:0:19896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c6ff lrc: 3/1,0 mode: --/CR res: [0x200000401:0x268:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.219017:0:19896:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c6ff lrc: 3/1,0 mode: --/CR res: [0x200000401:0x268:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764773.219039:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c9c00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966996480:192.168.202.146@tcp:101:rm.0 00000100:00100000:0.0:1775764773.219197:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f000 going to sleep for 11 seconds 00000100:00100000:0.0:1775764773.233082:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c9c00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966996480:192.168.202.146@tcp:101:rm.0 00010000:00010000:0.0:1775764773.233108:0:19896:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c6ff lrc: 4/1,0 mode: --/CR res: [0x200000401:0x268:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad27 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.233126:0:19896:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c6ff lrc: 4/1,0 mode: --/PR res: [0x200000401:0x269:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad27 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.233134:0:19896:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c6ff lrc: 4/1,0 mode: --/PR res: [0x200000401:0x269:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad27 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.233145:0:19896:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c6ff lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x269:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fad27 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.233154:0:19896:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c6ff lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x269:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad27 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.233165:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c6ff lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x269:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fad27 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.233173:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c6ff lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x269:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fad27 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.233180:0:19896:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c6ff lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x269:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fad27 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764773.233191:0:19896:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994f0c9c00 x1862022966996480/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/584 e 0 to 0 dl 1775764789 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1775764773.233204:0:19896:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c6ff lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x269:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad27 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.233213:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/617 (0 0) 00000080:00200000:0.0:1775764773.233219:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x268:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1775764773.233241:0:19896:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x269:0x0], updating i_size 7168 00000080:00010000:0.0:1775764773.233247:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x269:0x0](ffff8f9952db63c8) for lock 0x36f9bdc9e421c6ff 00000080:00010000:0.0:1775764773.233279:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x269:0x0](ffff8f9952db63c8) 00000080:00010000:0.0:1775764773.233283:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c6ff from it ffffafbac1ec7cb8 00010000:00010000:0.0:1775764773.233286:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c6ff lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x269:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fad27 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.233296:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c6ff lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x269:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad27 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764773.233680:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764773.233689:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=258 00000080:00200000:0.0:1775764773.233695:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 44 00000080:00200008:0.0:1775764773.233705:0:19896:0:(file.c:1057:ll_file_open()) START file client1:[0x200000401:0x269:0x0](ffff8f9952db63c8), flags 704400 00000002:00010000:0.0:1775764773.233727:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x269:0x0]) in obj [0x200000401:0x268:0x0], intent: open flags 0100240704401 00010000:00010000:0.0:1775764773.234046:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c706 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x268:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.234054:0:19896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c706 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x268:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1775764773.234066:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1775764773.234074:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f0c9c00 x1862022966996736/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/135824 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1775764773.234084:0:19896:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c706 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x268:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764773.234105:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c9c00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966996736:192.168.202.146@tcp:101:rm.0 00000100:00100000:0.0:1775764773.234287:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f000 going to sleep for 11 seconds 00000100:00100000:0.0:1775764773.243674:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c9c00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966996736:192.168.202.146@tcp:101:rm.0 00010000:00010000:0.0:1775764773.243700:0:19896:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c706 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x268:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.243714:0:19896:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c706 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x268:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.243721:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c706 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x268:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.243728:0:19896:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c706 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x268:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.243756:0:19896:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c706 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x268:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.243763:0:19896:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99821cc800 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:0.0:1775764773.243767:0:19896:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c706 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x268:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.243773:0:19896:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c706 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x268:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.243780:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c706 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x268:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.243788:0:19896:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1775764773.243793:0:19896:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c706 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x268:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.243799:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c706 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x268:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764773.243820:0:19896:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8f994f0c9c00 x1862022966996736/t4294997557(4294997557) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764789 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000002:00100000:0.0:1775764773.243833:0:19896:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f994f0c9c00 x1862022966996736/t4294997557(4294997557) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764789 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1775764773.243846:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0x268:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1775764773.243866:0:19896:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x269:0x0], updating i_size 7168 00000002:00100000:0.0:1775764773.243887:0:19896:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f994f0c9c00 x1862022966996736/t4294997557(4294997557) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764789 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200008:0.0:1775764773.243972:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file client1:[0x200000401:0x269:0x0](ffff8f9952db63c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764773.243978:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8),name=client1 00000002:00010000:0.0:1775764773.243998:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x269:0x0]) in obj [0x200000401:0x268:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764773.244013:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c6ff lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x269:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad27 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764773.244026:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c6ff lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x269:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad27 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764773.244033:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x268:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764773.244038:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x269:0x0](ffff8f9952db63c8) 00000080:00010000:0.0:1775764773.244042:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x269:0x0](ffff8f9952db63c8) for lock 0x36f9bdc9e421c6ff 00000080:00010000:0.0:1775764773.244047:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c6ff from it ffffafbac1ec7cd8 00010000:00010000:0.0:1775764773.244050:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c6ff lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x269:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fad27 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764773.244059:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c6ff lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x269:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad27 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764773.244094:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8) pos/size0/7168 32bit_api 0 00000002:00010000:0.0:1775764773.244107:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x269:0x0]) in obj [0x200000401:0x269:0x0], intent: readdir flags 0 00010000:00010000:0.0:1775764773.244114:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c6ff lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x269:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad27 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764773.244121:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c6ff lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x269:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad27 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000100:00100000:0.0:1775764773.246492:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c8e00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966996864:192.168.202.146@tcp:37:rm.0 00000100:00100000:0.0:1775764773.246590:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f000 going to sleep for 11 seconds 00000100:00100000:1.0:1775764773.268014:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c8e00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966996864:192.168.202.146@tcp:37:rm.0 00010000:00010000:1.0:1775764773.268584:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c6ff lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x269:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fad27 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764773.268611:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c6ff lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x269:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad27 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200008:1.0:1775764773.268670:0:19896:0:(file.c:447:ll_file_release()) START file client1:[0x200000401:0x269:0x0](ffff8f9952db63c8), flags 704000 00000002:00100000:1.0:1775764773.268730:0:19896:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f994f0c9c00 x1862022966996736/t4294997557(4294997557) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764789 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'rm.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1775764773.268764:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764773.268769:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e7b80 x1862022966997120/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764773.268790:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966997120:192.168.202.146@tcp:35:rm.0 00000100:00100000:1.0:1775764773.268875:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f997130d000 going to sleep for 11 seconds 00000100:00100000:1.0:1775764773.286109:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966997120:192.168.202.146@tcp:35:rm.0 00000002:00100000:1.0:1775764773.286144:0:19896:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f994f0c9c00 x1862022966996736/t4294997557(4294997557) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764789 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764773.286218:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764773.286227:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=258 00000080:00200000:1.0:1775764773.286235:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 44 00000080:00200008:1.0:1775764773.286244:0:19896:0:(file.c:1057:ll_file_open()) START file client1:[0x200000401:0x269:0x0](ffff8f9952db63c8), flags 704400 00000002:00010000:1.0:1775764773.286265:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x269:0x0]) in obj [0x200000401:0x268:0x0], intent: open flags 0104240704401 00010000:00010000:1.0:1775764773.286320:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c70d lrc: 3/1,0 mode: --/CR res: [0x200000401:0x268:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764773.286330:0:19896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c70d lrc: 3/1,0 mode: --/CR res: [0x200000401:0x268:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764773.286340:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764773.286346:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e7b80 x1862022966997376/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/135824 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764773.286355:0:19896:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c70d lrc: 3/1,0 mode: --/CR res: [0x200000401:0x268:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764773.286465:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966997376:192.168.202.146@tcp:101:rm.0 00000100:00100000:1.0:1775764773.286736:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f997130d000 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.298185:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966997376:192.168.202.146@tcp:101:rm.0 00010000:00010000:2.0:1775764773.298217:0:19896:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c70d lrc: 4/1,0 mode: --/CR res: [0x200000401:0x269:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad43 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.298226:0:19896:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c70d lrc: 4/1,0 mode: --/CR res: [0x200000401:0x269:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad43 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.298237:0:19896:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c70d lrc: 4/1,0 mode: CR/CR res: [0x200000401:0x269:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fad43 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.298260:0:19896:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c70d lrc: 4/1,0 mode: CR/CR res: [0x200000401:0x269:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad43 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764773.298273:0:19896:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=200002b, status=0 req@ffff8f994f2e7b80 x1862022966997376/t4294997559(4294997559) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764789 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764773.298286:0:19896:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f994f2e7b80 x1862022966997376/t4294997559(4294997559) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764789 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764773.298297:0:19896:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c70d lrc: 3/1,0 mode: CR/CR res: [0x200000401:0x269:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad43 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.298307:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 16 res 8589935617/617 (0 0) 00000080:00200000:2.0:1775764773.298313:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0x268:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764773.298333:0:19896:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x269:0x0], updating i_size 7168 00000080:00010000:2.0:1775764773.298338:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x269:0x0](ffff8f9952db63c8) for lock 0x36f9bdc9e421c70d 00000080:00010000:2.0:1775764773.298347:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c70d from it ffffafbac1ec7bd0 00010000:00010000:2.0:1775764773.298351:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c70d lrc: 3/1,0 mode: CR/CR res: [0x200000401:0x269:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fad43 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.298360:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c70d lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x269:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad43 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000002:00100000:2.0:1775764773.298399:0:19896:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f994f2e7b80 x1862022966997376/t4294997559(4294997559) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764789 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200008:2.0:1775764773.298469:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file client1:[0x200000401:0x269:0x0](ffff8f9952db63c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764773.298475:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8),name=client1 00000002:00010000:2.0:1775764773.298921:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x269:0x0]) in obj [0x200000401:0x268:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764773.298934:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c6ff lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x269:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad27 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764773.298946:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c6ff lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x269:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad27 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764773.298953:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x268:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764773.298959:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x269:0x0](ffff8f9952db63c8) 00000080:00010000:2.0:1775764773.298963:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x269:0x0](ffff8f9952db63c8) for lock 0x36f9bdc9e421c6ff 00000080:00010000:2.0:1775764773.298967:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c6ff from it ffffafbac1ec7cd8 00010000:00010000:2.0:1775764773.298971:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c6ff lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x269:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fad27 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764773.298979:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c6ff lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x269:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad27 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1775764773.299017:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8) pos/size0/7168 32bit_api 0 00000080:00200000:2.0:1775764773.299044:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8) pos/size18446744073709551614/7168 32bit_api 0 00000080:00200000:2.0:1775764773.299065:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:2.0:1775764773.299072:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=0 00000080:00200008:2.0:1775764773.299078:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file ~dmtmp:[0x200000401:0x26b:0x0](ffff8f9952db00c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764773.299082:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8),name=~dmtmp 00000002:00010000:2.0:1775764773.299089:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x26b:0x0]) in obj [0x200000401:0x269:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764773.299097:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9800/0x36f9bdc9e41cb202 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ef946 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764773.299106:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9800/0x36f9bdc9e41cb202 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ef946 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764773.299112:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x269:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764773.299116:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x26b:0x0](ffff8f9952db00c8) 00000080:00010000:2.0:1775764773.299119:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x26b:0x0](ffff8f9952db00c8) for lock 0x36f9bdc9e41cb202 00000080:00010000:2.0:1775764773.299122:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e41cb202 from it ffffafbac1ec7cb8 00010000:00010000:2.0:1775764773.299125:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9800/0x36f9bdc9e41cb202 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e4ef946 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764773.299132:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9800/0x36f9bdc9e41cb202 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ef946 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764773.299157:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:2.0:1775764773.299161:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=258 00000080:00200000:2.0:1775764773.299165:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 44 00000080:00200008:2.0:1775764773.299173:0:19896:0:(file.c:1057:ll_file_open()) START file ~dmtmp:[0x200000401:0x26b:0x0](ffff8f9952db00c8), flags 704400 00000080:00200008:2.0:1775764773.299185:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file ~dmtmp:[0x200000401:0x26b:0x0](ffff8f9952db00c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764773.299189:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8),name=~dmtmp 00000002:00010000:2.0:1775764773.299195:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x26b:0x0]) in obj [0x200000401:0x269:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764773.299201:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9800/0x36f9bdc9e41cb202 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ef946 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764773.299208:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9800/0x36f9bdc9e41cb202 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ef946 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764773.299213:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x269:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764773.299216:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x26b:0x0](ffff8f9952db00c8) 00000080:00010000:2.0:1775764773.299218:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x26b:0x0](ffff8f9952db00c8) for lock 0x36f9bdc9e41cb202 00000080:00010000:2.0:1775764773.299221:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e41cb202 from it ffffafbac1ec7cd8 00010000:00010000:2.0:1775764773.299224:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9800/0x36f9bdc9e41cb202 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e4ef946 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764773.299229:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9800/0x36f9bdc9e41cb202 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ef946 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764773.299241:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8) pos/size0/7168 32bit_api 0 00000002:00010000:2.0:1775764773.299250:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x26b:0x0]) in obj [0x200000401:0x26b:0x0], intent: readdir flags 0 00010000:00010000:2.0:1775764773.299254:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9800/0x36f9bdc9e41cb202 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ef946 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764773.299260:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9800/0x36f9bdc9e41cb202 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ef946 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000100:00100000:3.0:1775764773.303563:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966997504:192.168.202.146@tcp:37:rm.0 00000100:00100000:3.0:1775764773.303676:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413540 going to sleep for 11 seconds 00000100:00100000:3.0:1775764773.315396:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966997504:192.168.202.146@tcp:37:rm.0 00010000:00010000:3.0:1775764773.316343:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9800/0x36f9bdc9e41cb202 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e4ef946 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764773.316615:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9800/0x36f9bdc9e41cb202 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ef946 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200008:3.0:1775764773.316694:0:19896:0:(file.c:447:ll_file_release()) START file ~dmtmp:[0x200000401:0x26b:0x0](ffff8f9952db00c8), flags 704000 00010000:00010000:3.0:1775764773.316721:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9940030600/0x36f9bdc9e41ca521 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ed7e3 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764773.316790:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764773.316798:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=~dmtmp, flags=258 00000080:00200000:3.0:1775764773.316806:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 44 00000080:00200008:3.0:1775764773.316812:0:19896:0:(file.c:1057:ll_file_open()) START file ~dmtmp:[0x200000401:0x26b:0x0](ffff8f9952db00c8), flags 704400 00000080:00200008:3.0:1775764773.316831:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file ~dmtmp:[0x200000401:0x26b:0x0](ffff8f9952db00c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764773.316836:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8),name=~dmtmp 00000002:00010000:3.0:1775764773.316857:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x26b:0x0]) in obj [0x200000401:0x269:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764773.316865:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9800/0x36f9bdc9e41cb202 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ef946 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764773.316874:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9800/0x36f9bdc9e41cb202 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ef946 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764773.316881:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x269:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764773.316887:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x26b:0x0](ffff8f9952db00c8) 00000080:00010000:3.0:1775764773.316890:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x26b:0x0](ffff8f9952db00c8) for lock 0x36f9bdc9e41cb202 00000080:00010000:3.0:1775764773.316896:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e41cb202 from it ffffafbac1ec7cd8 00010000:00010000:3.0:1775764773.316899:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9800/0x36f9bdc9e41cb202 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e4ef946 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764773.316911:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9800/0x36f9bdc9e41cb202 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ef946 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764773.316949:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8) pos/size0/7168 32bit_api 0 00000080:00200000:3.0:1775764773.316974:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8) pos/size18446744073709551614/7168 32bit_api 0 00000080:00200000:3.0:1775764773.316985:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764773.316990:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=SEED, flags=0 00000080:00200008:3.0:1775764773.316995:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file SEED:[0x200000401:0x2e5:0x0](ffff8f9952dd2a08), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764773.316998:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2e5:0x0](ffff8f9952dd2a08),name=SEED 00000002:00010000:3.0:1775764773.317006:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2e5:0x0]) in obj [0x200000401:0x26b:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764773.317012:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/741 (0 0) 00010000:00010000:3.0:1775764773.317056:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c714 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.317064:0:19896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c714 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.317074:0:19896:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c714 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764773.317101:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966997632:192.168.202.146@tcp:101:rm.0 00000100:00100000:3.0:1775764773.318164:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413540 going to sleep for 11 seconds 00000100:00100000:3.0:1775764773.325267:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966997632:192.168.202.146@tcp:101:rm.0 00010000:00010000:3.0:1775764773.325292:0:19896:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c714 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad58 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.325314:0:19896:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c714 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad58 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.325322:0:19896:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c714 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad58 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.325334:0:19896:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c714 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fad58 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.325346:0:19896:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c714 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad58 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.325358:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c714 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fad58 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.325377:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c714 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fad58 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.325384:0:19896:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c714 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fad58 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764773.325395:0:19896:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970ede300 x1862022966997632/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/584 e 0 to 0 dl 1775764789 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1775764773.325407:0:19896:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c714 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad58 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.325416:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/741 (0 0) 00000080:00200000:3.0:1775764773.325423:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26b:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1775764773.325443:0:19896:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x2e5:0x0], updating i_size 7168 00000080:00010000:3.0:1775764773.325449:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2e5:0x0](ffff8f9952dd2a08) for lock 0x36f9bdc9e421c714 00000080:00010000:3.0:1775764773.325457:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2e5:0x0](ffff8f9952dd2a08) 00000080:00010000:3.0:1775764773.325460:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c714 from it ffffafbac1ec7cb8 00010000:00010000:3.0:1775764773.325463:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c714 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fad58 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.325472:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c714 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad58 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764773.325552:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764773.325561:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=SEED, flags=258 00000080:00200000:3.0:1775764773.325566:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2e5:0x0](ffff8f9952dd2a08), inode mode 41c0 mask 44 00000080:00200008:3.0:1775764773.325574:0:19896:0:(file.c:1057:ll_file_open()) START file SEED:[0x200000401:0x2e5:0x0](ffff8f9952dd2a08), flags 704400 00000080:00200008:3.0:1775764773.325604:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file SEED:[0x200000401:0x2e5:0x0](ffff8f9952dd2a08), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764773.325610:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2e5:0x0](ffff8f9952dd2a08),name=SEED 00000002:00010000:3.0:1775764773.325628:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2e5:0x0]) in obj [0x200000401:0x26b:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764773.325638:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c714 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad58 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764773.325648:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c714 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad58 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764773.325655:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26b:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764773.325659:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2e5:0x0](ffff8f9952dd2a08) 00000080:00010000:3.0:1775764773.325662:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2e5:0x0](ffff8f9952dd2a08) for lock 0x36f9bdc9e421c714 00000080:00010000:3.0:1775764773.325665:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c714 from it ffffafbac1ec7cd8 00010000:00010000:3.0:1775764773.325669:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c714 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fad58 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764773.325676:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c714 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad58 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764773.325700:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2e5:0x0](ffff8f9952dd2a08) pos/size0/7168 32bit_api 0 00000002:00010000:3.0:1775764773.325712:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2e5:0x0]) in obj [0x200000401:0x2e5:0x0], intent: readdir flags 0 00010000:00010000:3.0:1775764773.325718:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c714 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad58 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764773.325725:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c714 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad58 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000100:00100000:3.0:1775764773.326516:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966997760:192.168.202.146@tcp:37:rm.0 00000100:00100000:3.0:1775764773.326616:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413540 going to sleep for 11 seconds 00000100:00100000:3.0:1775764773.335141:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966997760:192.168.202.146@tcp:37:rm.0 00010000:00010000:3.0:1775764773.335845:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c714 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fad58 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764773.335862:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c714 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad58 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764773.335907:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2e5:0x0](ffff8f9952dd2a08) pos/size18446744073709551614/7168 32bit_api 0 00000080:00200008:3.0:1775764773.336336:0:19896:0:(file.c:447:ll_file_release()) START file SEED:[0x200000401:0x2e5:0x0](ffff8f9952dd2a08), flags 704000 00010000:00010000:3.0:1775764773.336536:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e41cb0ff lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2e5:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ef667 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1775764773.336583:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764773.336594:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=SEED, flags=0 00000080:00200000:3.0:1775764773.337094:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 3 00000080:00200000:3.0:1775764773.337103:0:19896:0:(namei.c:2305:ll_rmdir()) VFS Op:name=SEED, dir=[0x200000401:0x26b:0x0](ffff8f9952db00c8) 00010000:00010000:3.0:1775764773.337134:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9800/0x36f9bdc9e41cb202 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e4ef946 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764773.337152:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9940030600/0x36f9bdc9e41ca521 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ed7e3 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764773.337239:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c714 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fad58 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764773.337287:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c283 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x11/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f8d0d expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764773.337299:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e41cb0ff lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2e5:0x0].0x0 bits 0x5/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e4ef667 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000002:00100000:3.0:1775764773.337381:0:19896:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f9970f6aa00 x1862022950425600/t4294969758(4294969758) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764075 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000020:00100000:3.0:1775764773.337394:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:3.0:1775764773.337400:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970ede300 x1862022966998016/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:3.0:1775764773.337425:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966998016:192.168.202.146@tcp:35:rm.0 00000100:00100000:3.0:1775764773.337607:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413540 going to sleep for 11 seconds 00000100:00100000:3.0:1775764773.341790:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966998016:192.168.202.146@tcp:35:rm.0 00000002:00100000:3.0:1775764773.341823:0:19896:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f9970f6aa00 x1862022950425600/t4294969758(4294969758) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764075 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00010000:00010000:3.0:1775764773.341962:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c714 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fad58 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764773.341973:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c283 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x11/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8d0d expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764773.341980:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e41cb0ff lrc: 1/0,0 mode: --/CR res: [0x200000401:0x2e5:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ef667 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1775764773.341987:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9800/0x36f9bdc9e41cb202 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ef946 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764773.341993:0:19896:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 4 locks packed 00010000:00010000:3.0:1775764773.341996:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c714 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fad58 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764773.342008:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c283 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2e5:0x0].0x0 bits 0x11/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8d0d expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764773.342017:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e41cb0ff lrc: 0/0,0 mode: --/CR res: [0x200000401:0x2e5:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ef667 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1775764773.342037:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9800/0x36f9bdc9e41cb202 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ef946 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000020:00100000:3.0:1775764773.342060:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1775764773.342066:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970ede300 x1862022966998272/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 504/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764773.342089:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966998272:192.168.202.146@tcp:36:rm.0 00000100:00100000:3.0:1775764773.342173:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413540 going to sleep for 11 seconds 00000100:00100000:3.0:1775764773.351300:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966998272:192.168.202.146@tcp:36:rm.0 00000080:00200000:3.0:1775764773.351345:0:19896:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2e5:0x0](ffff8f9952dd2a08) 00000080:00200000:3.0:1775764773.351437:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764773.351445:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=0 00000080:00200008:3.0:1775764773.351456:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file PARADOX:[0x200000401:0x2b8:0x0](ffff8f9952d880c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764773.351461:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8),name=PARADOX 00000002:00010000:3.0:1775764773.351483:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2b8:0x0]) in obj [0x200000401:0x26b:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764773.351507:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/696 (0 0) 00010000:00010000:3.0:1775764773.351561:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c71b lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.351571:0:19896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c71b lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.351582:0:19896:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c71b lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764773.351613:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966998400:192.168.202.146@tcp:101:rm.0 00000100:00100000:3.0:1775764773.351717:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413540 going to sleep for 11 seconds 00000100:00100000:3.0:1775764773.355456:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966998400:192.168.202.146@tcp:101:rm.0 00010000:00010000:3.0:1775764773.355485:0:19896:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c71b lrc: 4/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad7b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.355513:0:19896:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c71b lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad7b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.355521:0:19896:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c71b lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad7b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.355533:0:19896:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c71b lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fad7b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.355543:0:19896:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c71b lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad7b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.355554:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c71b lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fad7b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.355562:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c71b lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fad7b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.355569:0:19896:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c71b lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fad7b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764773.355580:0:19896:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970edc000 x1862022966998400/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/584 e 0 to 0 dl 1775764789 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1775764773.355591:0:19896:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c71b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad7b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.355614:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/696 (0 0) 00000080:00200000:3.0:1775764773.355620:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26b:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1775764773.355640:0:19896:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x2b8:0x0], updating i_size 7168 00000080:00010000:3.0:1775764773.355645:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2b8:0x0](ffff8f9952d880c8) for lock 0x36f9bdc9e421c71b 00000080:00010000:3.0:1775764773.355654:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2b8:0x0](ffff8f9952d880c8) 00000080:00010000:3.0:1775764773.355657:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c71b from it ffffafbac1ec7cb8 00010000:00010000:3.0:1775764773.355660:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c71b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fad7b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.355668:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c71b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad7b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764773.362100:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:2.0:1775764773.362117:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=258 00000080:00200000:2.0:1775764773.362126:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8), inode mode 41c0 mask 44 00000080:00200008:2.0:1775764773.362134:0:19896:0:(file.c:1057:ll_file_open()) START file PARADOX:[0x200000401:0x2b8:0x0](ffff8f9952d880c8), flags 704400 00000080:00200008:2.0:1775764773.362167:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file PARADOX:[0x200000401:0x2b8:0x0](ffff8f9952d880c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764773.362172:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8),name=PARADOX 00000002:00010000:2.0:1775764773.362194:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2b8:0x0]) in obj [0x200000401:0x26b:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764773.362213:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c71b lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad7b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764773.362228:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c71b lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad7b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764773.362235:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26b:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764773.362242:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2b8:0x0](ffff8f9952d880c8) 00000080:00010000:2.0:1775764773.362245:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2b8:0x0](ffff8f9952d880c8) for lock 0x36f9bdc9e421c71b 00000080:00010000:2.0:1775764773.362251:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c71b from it ffffafbac1ec7cd8 00010000:00010000:2.0:1775764773.362254:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c71b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fad7b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764773.362262:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c71b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad7b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764773.362294:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8) pos/size0/7168 32bit_api 0 00000002:00010000:2.0:1775764773.362306:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2b8:0x0]) in obj [0x200000401:0x2b8:0x0], intent: readdir flags 0 00010000:00010000:2.0:1775764773.362312:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c71b lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad7b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764773.362321:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c71b lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad7b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000100:00100000:2.0:1775764773.365332:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdd180 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966998528:192.168.202.146@tcp:37:rm.0 00000100:00100000:2.0:1775764773.368813:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102000 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.380374:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdd180 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966998528:192.168.202.146@tcp:37:rm.0 00010000:00010000:2.0:1775764773.381015:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c71b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fad7b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764773.381030:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c71b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad7b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764773.381071:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8) pos/size18446744073709551614/7168 32bit_api 0 00000080:00200008:2.0:1775764773.381088:0:19896:0:(file.c:447:ll_file_release()) START file PARADOX:[0x200000401:0x2b8:0x0](ffff8f9952d880c8), flags 704000 00010000:00010000:2.0:1775764773.381109:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e41caad1 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ee710 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764773.381139:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:2.0:1775764773.381148:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PARADOX, flags=0 00000080:00200000:2.0:1775764773.381154:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 3 00000080:00200000:2.0:1775764773.381158:0:19896:0:(namei.c:2305:ll_rmdir()) VFS Op:name=PARADOX, dir=[0x200000401:0x26b:0x0](ffff8f9952db00c8) 00010000:00010000:2.0:1775764773.381183:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c71b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fad7b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764773.381249:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970abee00/0x36f9bdc9e41cab09 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x11/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e4ee795 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 79 lru_type: 1 00010000:00010000:2.0:1775764773.381265:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e41caad1 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x5/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e4ee710 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000002:00100000:2.0:1775764773.381308:0:19896:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f994f2e6300 x1862022950364416/t4294969658(4294969658) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764072 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764773.381319:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764773.381323:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994efdd180 x1862022966998784/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764773.381344:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdd180 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966998784:192.168.202.146@tcp:35:rm.0 00000100:00100000:2.0:1775764773.381431:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102000 going to sleep for 11 seconds 00010000:00010000:2.0:1775764773.384684:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764773.385933:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdd180 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966998784:192.168.202.146@tcp:35:rm.0 00000002:00100000:2.0:1775764773.385968:0:19896:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f994f2e6300 x1862022950364416/t4294969658(4294969658) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764072 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00010000:00010000:2.0:1775764773.386067:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c71b lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fad7b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764773.386078:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970abee00/0x36f9bdc9e41cab09 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x11/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ee795 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 79 lru_type: 1 00010000:00010000:2.0:1775764773.386085:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e41caad1 lrc: 1/0,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ee710 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764773.386092:0:19896:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:2.0:1775764773.386094:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5600/0x36f9bdc9e421c71b lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fad7b expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764773.386106:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970abee00/0x36f9bdc9e41cab09 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2b8:0x0].0x0 bits 0x11/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ee795 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 79 lru_type: 1 00010000:00010000:2.0:1775764773.386113:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e41caad1 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x2b8:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ee710 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000020:00100000:2.0:1775764773.386134:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1775764773.386141:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994efdd180 x1862022966999040/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764773.386163:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdd180 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966999040:192.168.202.146@tcp:36:rm.0 00000100:00100000:2.0:1775764773.386254:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102000 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.397660:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdd180 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966999040:192.168.202.146@tcp:36:rm.0 00000080:00200000:2.0:1775764773.397706:0:19896:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2b8:0x0](ffff8f9952d880c8) 00000080:00200000:2.0:1775764773.397824:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:2.0:1775764773.397835:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=EXCEL, flags=0 00000080:00200008:2.0:1775764773.397848:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file EXCEL:[0x200000401:0x281:0x0](ffff8f9952dbd348), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764773.397854:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x281:0x0](ffff8f9952dbd348),name=EXCEL 00000002:00010000:2.0:1775764773.397878:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x281:0x0]) in obj [0x200000401:0x26b:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764773.397902:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/641 (0 0) 00010000:00010000:2.0:1775764773.397957:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c722 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.397967:0:19896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c722 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.397981:0:19896:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c722 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764773.398004:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efddf80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966999168:192.168.202.146@tcp:101:rm.0 00000100:00100000:2.0:1775764773.398123:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102000 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.409896:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efddf80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966999168:192.168.202.146@tcp:101:rm.0 00010000:00010000:2.0:1775764773.410466:0:19896:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c722 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad9e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.410575:0:19896:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c722 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x281:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad9e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.410586:0:19896:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c722 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x281:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad9e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.410607:0:19896:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c722 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x281:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fad9e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.410868:0:19896:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c722 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x281:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad9e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.410882:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c722 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x281:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fad9e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.410889:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c722 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x281:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fad9e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.410902:0:19896:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c722 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x281:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fad9e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764773.410915:0:19896:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994efddf80 x1862022966999168/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/584 e 0 to 0 dl 1775764789 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764773.411000:0:19896:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c722 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x281:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad9e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.411011:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/641 (0 0) 00000080:00200000:2.0:1775764773.411019:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26b:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764773.411043:0:19896:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x281:0x0], updating i_size 7168 00000080:00010000:2.0:1775764773.411048:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x281:0x0](ffff8f9952dbd348) for lock 0x36f9bdc9e421c722 00000080:00010000:0.0:1775764773.424170:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x281:0x0](ffff8f9952dbd348) 00000080:00010000:0.0:1775764773.424180:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c722 from it ffffafbac1ec7cb8 00010000:00010000:0.0:1775764773.424189:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c722 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x281:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fad9e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.424201:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c722 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x281:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad9e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764773.424305:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764773.424313:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=EXCEL, flags=258 00000080:00200000:0.0:1775764773.424321:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x281:0x0](ffff8f9952dbd348), inode mode 41c0 mask 44 00000080:00200008:0.0:1775764773.424329:0:19896:0:(file.c:1057:ll_file_open()) START file EXCEL:[0x200000401:0x281:0x0](ffff8f9952dbd348), flags 704400 00000080:00200008:0.0:1775764773.424348:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file EXCEL:[0x200000401:0x281:0x0](ffff8f9952dbd348), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764773.424353:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x281:0x0](ffff8f9952dbd348),name=EXCEL 00000002:00010000:0.0:1775764773.424386:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x281:0x0]) in obj [0x200000401:0x26b:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764773.424402:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c722 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x281:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad9e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764773.424413:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c722 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x281:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad9e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764773.424421:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26b:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764773.424427:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x281:0x0](ffff8f9952dbd348) 00000080:00010000:0.0:1775764773.424431:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x281:0x0](ffff8f9952dbd348) for lock 0x36f9bdc9e421c722 00000080:00010000:0.0:1775764773.424436:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c722 from it ffffafbac1ec7cd8 00010000:00010000:0.0:1775764773.424439:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c722 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x281:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fad9e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764773.424446:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c722 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x281:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad9e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764773.424469:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x281:0x0](ffff8f9952dbd348) pos/size0/7168 32bit_api 0 00000002:00010000:0.0:1775764773.424481:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x281:0x0]) in obj [0x200000401:0x281:0x0], intent: readdir flags 0 00010000:00010000:0.0:1775764773.424499:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c722 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x281:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad9e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764773.424508:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c722 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x281:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad9e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000100:00100000:0.0:1775764773.425343:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0cb480 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966999296:192.168.202.146@tcp:37:rm.0 00000100:00100000:0.0:1775764773.425473:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f240 going to sleep for 11 seconds 00000100:00100000:0.0:1775764773.434958:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0cb480 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966999296:192.168.202.146@tcp:37:rm.0 00010000:00010000:0.0:1775764773.435563:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c722 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x281:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fad9e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764773.435579:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c722 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x281:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad9e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1775764773.435637:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x281:0x0](ffff8f9952dbd348) pos/size18446744073709551614/7168 32bit_api 0 00000080:00200008:0.0:1775764773.435657:0:19896:0:(file.c:447:ll_file_release()) START file EXCEL:[0x200000401:0x281:0x0](ffff8f9952dbd348), flags 704000 00010000:00010000:0.0:1775764773.435684:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5400/0x36f9bdc9e41ca2dc lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x281:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ed202 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00000080:00200000:0.0:1775764773.435720:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764773.435730:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=EXCEL, flags=0 00000080:00200000:0.0:1775764773.435736:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 3 00000080:00200000:0.0:1775764773.435782:0:19896:0:(namei.c:2305:ll_rmdir()) VFS Op:name=EXCEL, dir=[0x200000401:0x26b:0x0](ffff8f9952db00c8) 00010000:00010000:0.0:1775764773.435808:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c722 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x281:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fad9e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764773.435880:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971071c00/0x36f9bdc9e41ca31b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x281:0x0].0x0 bits 0x11/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e4ed2aa expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:0.0:1775764773.435893:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5400/0x36f9bdc9e41ca2dc lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x281:0x0].0x0 bits 0x5/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e4ed202 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00000002:00100000:0.0:1775764773.435943:0:19896:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f9970f68000 x1862022950284032/t4294969522(4294969522) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764067 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1775764773.435954:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1775764773.435959:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f0cb480 x1862022966999552/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1775764773.435983:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0cb480 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966999552:192.168.202.146@tcp:35:rm.0 00000100:00100000:0.0:1775764773.436070:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f240 going to sleep for 11 seconds 00000100:00100000:0.0:1775764773.439246:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0cb480 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966999552:192.168.202.146@tcp:35:rm.0 00000002:00100000:0.0:1775764773.439278:0:19896:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f9970f68000 x1862022950284032/t4294969522(4294969522) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764067 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00010000:00010000:0.0:1775764773.439348:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c722 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x281:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fad9e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764773.439360:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971071c00/0x36f9bdc9e41ca31b lrc: 1/0,0 mode: --/PR res: [0x200000401:0x281:0x0].0x0 bits 0x11/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ed2aa expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:0.0:1775764773.439378:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5400/0x36f9bdc9e41ca2dc lrc: 1/0,0 mode: --/CR res: [0x200000401:0x281:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ed202 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00010000:00010000:0.0:1775764773.439384:0:19896:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:0.0:1775764773.439387:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c722 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x281:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fad9e expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764773.439397:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971071c00/0x36f9bdc9e41ca31b lrc: 0/0,0 mode: --/PR res: [0x200000401:0x281:0x0].0x0 bits 0x11/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ed2aa expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:0.0:1775764773.439405:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5400/0x36f9bdc9e41ca2dc lrc: 0/0,0 mode: --/CR res: [0x200000401:0x281:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ed202 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00000020:00100000:0.0:1775764773.439425:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1775764773.439432:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f0cb480 x1862022966999808/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764773.439451:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0cb480 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966999808:192.168.202.146@tcp:36:rm.0 00000100:00100000:0.0:1775764773.439529:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f240 going to sleep for 11 seconds 00000100:00100000:0.0:1775764773.443815:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0cb480 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966999808:192.168.202.146@tcp:36:rm.0 00000080:00200000:0.0:1775764773.443855:0:19896:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x281:0x0](ffff8f9952dbd348) 00000080:00200000:0.0:1775764773.443948:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764773.443958:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=WORD, flags=0 00000080:00200008:0.0:1775764773.443969:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file WORD:[0x200000401:0x278:0x0](ffff8f9952dbf448), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764773.443974:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x278:0x0](ffff8f9952dbf448),name=WORD 00000002:00010000:0.0:1775764773.443994:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x278:0x0]) in obj [0x200000401:0x26b:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764773.444005:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/632 (0 0) 00010000:00010000:0.0:1775764773.444053:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c729 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.444062:0:19896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c729 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.444071:0:19896:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c729 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764773.444090:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0caa00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966999936:192.168.202.146@tcp:101:rm.0 00000100:00100000:0.0:1775764773.444170:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f240 going to sleep for 11 seconds 00000100:00100000:0.0:1775764773.452523:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0caa00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022966999936:192.168.202.146@tcp:101:rm.0 00010000:00010000:0.0:1775764773.452551:0:19896:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c729 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fadc1 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.452568:0:19896:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c729 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x278:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fadc1 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.452576:0:19896:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c729 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x278:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fadc1 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.452587:0:19896:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c729 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x278:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fadc1 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.452607:0:19896:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c729 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x278:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fadc1 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.452633:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c729 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x278:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fadc1 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.452640:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c729 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x278:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fadc1 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.452647:0:19896:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c729 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x278:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fadc1 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764773.452658:0:19896:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994f0caa00 x1862022966999936/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/584 e 0 to 0 dl 1775764789 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1775764773.452669:0:19896:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c729 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x278:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fadc1 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.452678:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/632 (0 0) 00000080:00200000:0.0:1775764773.452685:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26b:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1775764773.452706:0:19896:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x278:0x0], updating i_size 7168 00000080:00010000:0.0:1775764773.452711:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x278:0x0](ffff8f9952dbf448) for lock 0x36f9bdc9e421c729 00000080:00010000:0.0:1775764773.452722:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x278:0x0](ffff8f9952dbf448) 00000080:00010000:0.0:1775764773.452724:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c729 from it ffffafbac1ec7cb8 00010000:00010000:0.0:1775764773.452727:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c729 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x278:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fadc1 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.452735:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c729 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x278:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fadc1 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764773.452884:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764773.452895:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=WORD, flags=258 00000080:00200000:0.0:1775764773.452902:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x278:0x0](ffff8f9952dbf448), inode mode 41c0 mask 44 00000080:00200008:0.0:1775764773.452911:0:19896:0:(file.c:1057:ll_file_open()) START file WORD:[0x200000401:0x278:0x0](ffff8f9952dbf448), flags 704400 00000080:00200008:0.0:1775764773.452945:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file WORD:[0x200000401:0x278:0x0](ffff8f9952dbf448), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764773.452951:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x278:0x0](ffff8f9952dbf448),name=WORD 00000002:00010000:0.0:1775764773.452968:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x278:0x0]) in obj [0x200000401:0x26b:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764773.452977:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c729 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x278:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fadc1 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764773.452987:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c729 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x278:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fadc1 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764773.452992:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26b:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764773.452997:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x278:0x0](ffff8f9952dbf448) 00000080:00010000:0.0:1775764773.452999:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x278:0x0](ffff8f9952dbf448) for lock 0x36f9bdc9e421c729 00000080:00010000:0.0:1775764773.453002:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c729 from it ffffafbac1ec7cd8 00010000:00010000:0.0:1775764773.453005:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c729 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x278:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fadc1 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764773.453012:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c729 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x278:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fadc1 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764773.453033:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x278:0x0](ffff8f9952dbf448) pos/size0/7168 32bit_api 0 00000002:00010000:0.0:1775764773.453045:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x278:0x0]) in obj [0x200000401:0x278:0x0], intent: readdir flags 0 00010000:00010000:0.0:1775764773.453051:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c729 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x278:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fadc1 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764773.453059:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c729 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x278:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fadc1 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000100:00100000:0.0:1775764773.453632:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0caa00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967000064:192.168.202.146@tcp:37:rm.0 00000100:00100000:0.0:1775764773.453720:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f240 going to sleep for 11 seconds 00000100:00100000:0.0:1775764773.465043:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0caa00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967000064:192.168.202.146@tcp:37:rm.0 00010000:00010000:0.0:1775764773.465671:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c729 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x278:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fadc1 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764773.465687:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c729 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x278:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fadc1 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1775764773.465735:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x278:0x0](ffff8f9952dbf448) pos/size18446744073709551614/7168 32bit_api 0 00000080:00200008:0.0:1775764773.465804:0:19896:0:(file.c:447:ll_file_release()) START file WORD:[0x200000401:0x278:0x0](ffff8f9952dbf448), flags 704000 00010000:00010000:0.0:1775764773.465830:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5e00/0x36f9bdc9e41ca1a1 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x278:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ece9e expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 82 lru_type: 1 00000080:00200000:0.0:1775764773.465863:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764773.465876:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=WORD, flags=0 00000080:00200000:0.0:1775764773.465884:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 3 00000080:00200000:0.0:1775764773.465891:0:19896:0:(namei.c:2305:ll_rmdir()) VFS Op:name=WORD, dir=[0x200000401:0x26b:0x0](ffff8f9952db00c8) 00010000:00010000:0.0:1775764773.465915:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c729 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x278:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fadc1 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764773.465984:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5800/0x36f9bdc9e41ca1e7 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x278:0x0].0x0 bits 0x11/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e4ecf62 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00010000:00010000:0.0:1775764773.466002:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5e00/0x36f9bdc9e41ca1a1 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x278:0x0].0x0 bits 0x5/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e4ece9e expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 82 lru_type: 1 00000002:00100000:0.0:1775764773.466085:0:19896:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f9970f69c00 x1862022950271360/t4294969497(4294969497) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764067 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1775764773.466098:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1775764773.466103:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f0caa00 x1862022967000320/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1775764773.466128:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0caa00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967000320:192.168.202.146@tcp:35:rm.0 00000100:00100000:0.0:1775764773.466206:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f240 going to sleep for 11 seconds 00000100:00100000:0.0:1775764773.486560:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0caa00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967000320:192.168.202.146@tcp:35:rm.0 00000002:00100000:0.0:1775764773.486623:0:19896:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f9970f69c00 x1862022950271360/t4294969497(4294969497) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764067 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00010000:00010000:0.0:1775764773.486727:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c729 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x278:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fadc1 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764773.486776:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5800/0x36f9bdc9e41ca1e7 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x278:0x0].0x0 bits 0x11/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ecf62 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00010000:00010000:0.0:1775764773.486784:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5e00/0x36f9bdc9e41ca1a1 lrc: 1/0,0 mode: --/CR res: [0x200000401:0x278:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ece9e expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 82 lru_type: 1 00010000:00010000:0.0:1775764773.486791:0:19896:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:0.0:1775764773.486794:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c729 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x278:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fadc1 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764773.486811:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5800/0x36f9bdc9e41ca1e7 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x278:0x0].0x0 bits 0x11/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ecf62 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00010000:00010000:0.0:1775764773.486819:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970dc5e00/0x36f9bdc9e41ca1a1 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x278:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ece9e expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 82 lru_type: 1 00000020:00100000:0.0:1775764773.486840:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1775764773.486847:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f0caa00 x1862022967000576/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764773.486870:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0caa00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967000576:192.168.202.146@tcp:36:rm.0 00000100:00100000:2.0:1775764773.487280:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f240 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.506499:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0caa00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967000576:192.168.202.146@tcp:36:rm.0 00000080:00200000:2.0:1775764773.506547:0:19896:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x278:0x0](ffff8f9952dbf448) 00000080:00200000:2.0:1775764773.506845:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:2.0:1775764773.506856:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PWRPNT, flags=0 00000080:00200008:2.0:1775764773.506868:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file PWRPNT:[0x200000401:0x287:0x0](ffff8f9952dba1c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764773.506874:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x287:0x0](ffff8f9952dba1c8),name=PWRPNT 00000002:00010000:2.0:1775764773.506897:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x287:0x0]) in obj [0x200000401:0x26b:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764773.506909:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/647 (0 0) 00010000:00010000:2.0:1775764773.506958:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c730 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.506968:0:19896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c730 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.506978:0:19896:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c730 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764773.506998:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdf100 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967000704:192.168.202.146@tcp:101:rm.0 00000100:00100000:2.0:1775764773.507086:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102000 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.512652:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdf100 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967000704:192.168.202.146@tcp:101:rm.0 00010000:00010000:2.0:1775764773.512680:0:19896:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c730 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fade4 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.512697:0:19896:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c730 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x287:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fade4 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.512705:0:19896:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c730 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x287:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fade4 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.512716:0:19896:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c730 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x287:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fade4 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.512726:0:19896:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c730 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x287:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fade4 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.512769:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c730 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x287:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fade4 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.512778:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c730 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x287:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fade4 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.512788:0:19896:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c730 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x287:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fade4 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764773.512799:0:19896:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994efdf100 x1862022967000704/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/584 e 0 to 0 dl 1775764789 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764773.512810:0:19896:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c730 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x287:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fade4 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.512820:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/647 (0 0) 00000080:00200000:2.0:1775764773.512826:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26b:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764773.512848:0:19896:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x287:0x0], updating i_size 7168 00000080:00010000:2.0:1775764773.512853:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x287:0x0](ffff8f9952dba1c8) for lock 0x36f9bdc9e421c730 00000080:00010000:2.0:1775764773.512865:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x287:0x0](ffff8f9952dba1c8) 00000080:00010000:2.0:1775764773.512868:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c730 from it ffffafbac1ec7cb8 00010000:00010000:2.0:1775764773.512872:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c730 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x287:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fade4 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.512880:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c730 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x287:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fade4 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764773.512963:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:2.0:1775764773.512972:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PWRPNT, flags=258 00000080:00200000:2.0:1775764773.512978:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x287:0x0](ffff8f9952dba1c8), inode mode 41c0 mask 44 00000080:00200008:2.0:1775764773.512986:0:19896:0:(file.c:1057:ll_file_open()) START file PWRPNT:[0x200000401:0x287:0x0](ffff8f9952dba1c8), flags 704400 00000080:00200008:2.0:1775764773.513005:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file PWRPNT:[0x200000401:0x287:0x0](ffff8f9952dba1c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764773.513010:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x287:0x0](ffff8f9952dba1c8),name=PWRPNT 00000002:00010000:2.0:1775764773.513030:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x287:0x0]) in obj [0x200000401:0x26b:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764773.513039:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c730 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x287:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fade4 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764773.513050:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c730 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x287:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fade4 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764773.513056:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26b:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764773.513061:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x287:0x0](ffff8f9952dba1c8) 00000080:00010000:2.0:1775764773.513063:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x287:0x0](ffff8f9952dba1c8) for lock 0x36f9bdc9e421c730 00000080:00010000:2.0:1775764773.513067:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c730 from it ffffafbac1ec7cd8 00010000:00010000:2.0:1775764773.513070:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c730 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x287:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fade4 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764773.513076:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c730 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x287:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fade4 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764773.513100:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x287:0x0](ffff8f9952dba1c8) pos/size0/7168 32bit_api 0 00000002:00010000:2.0:1775764773.513112:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x287:0x0]) in obj [0x200000401:0x287:0x0], intent: readdir flags 0 00010000:00010000:2.0:1775764773.513119:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c730 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x287:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fade4 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764773.513127:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c730 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x287:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fade4 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000100:00100000:2.0:1775764773.514178:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdf100 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967000832:192.168.202.146@tcp:37:rm.0 00000100:00100000:2.0:1775764773.514281:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102000 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.532161:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdf100 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967000832:192.168.202.146@tcp:37:rm.0 00010000:00010000:2.0:1775764773.532838:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c730 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x287:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fade4 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764773.532856:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c730 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x287:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fade4 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764773.532907:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x287:0x0](ffff8f9952dba1c8) pos/size18446744073709551614/7168 32bit_api 0 00000080:00200008:2.0:1775764773.532927:0:19896:0:(file.c:447:ll_file_release()) START file PWRPNT:[0x200000401:0x287:0x0](ffff8f9952dba1c8), flags 704000 00010000:00010000:2.0:1775764773.532952:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3b9c00/0x36f9bdc9e41ca3d1 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x287:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ed4c5 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 160 lru_type: 1 00000080:00200000:2.0:1775764773.532990:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:2.0:1775764773.533000:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PWRPNT, flags=0 00000080:00200000:2.0:1775764773.533008:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 3 00000080:00200000:2.0:1775764773.533015:0:19896:0:(namei.c:2305:ll_rmdir()) VFS Op:name=PWRPNT, dir=[0x200000401:0x26b:0x0](ffff8f9952db00c8) 00010000:00010000:2.0:1775764773.533044:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c730 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x287:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fade4 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764773.533116:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3b8400/0x36f9bdc9e41ca409 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x287:0x0].0x0 bits 0x11/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e4ed54a expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 82 lru_type: 1 00010000:00010000:2.0:1775764773.533129:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3b9c00/0x36f9bdc9e41ca3d1 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x287:0x0].0x0 bits 0x5/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e4ed4c5 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 160 lru_type: 1 00000002:00100000:2.0:1775764773.533174:0:19896:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f99463c0a80 x1862022950293888/t4294969543(4294969543) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764068 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764773.533186:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764773.533191:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994efdf100 x1862022967001088/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764773.533214:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdf100 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967001088:192.168.202.146@tcp:35:rm.0 00000100:00100000:2.0:1775764773.533310:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102000 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.546511:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdf100 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967001088:192.168.202.146@tcp:35:rm.0 00000002:00100000:2.0:1775764773.546544:0:19896:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f99463c0a80 x1862022950293888/t4294969543(4294969543) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764068 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00010000:00010000:2.0:1775764773.546633:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c730 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x287:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fade4 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764773.546644:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3b8400/0x36f9bdc9e41ca409 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x287:0x0].0x0 bits 0x11/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ed54a expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 82 lru_type: 1 00010000:00010000:2.0:1775764773.546650:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3b9c00/0x36f9bdc9e41ca3d1 lrc: 1/0,0 mode: --/CR res: [0x200000401:0x287:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ed4c5 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 160 lru_type: 1 00010000:00010000:2.0:1775764773.546655:0:19896:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:2.0:1775764773.546658:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c730 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x287:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fade4 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764773.546669:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3b8400/0x36f9bdc9e41ca409 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x287:0x0].0x0 bits 0x11/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ed54a expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 82 lru_type: 1 00010000:00010000:2.0:1775764773.546676:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3b9c00/0x36f9bdc9e41ca3d1 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x287:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ed4c5 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 160 lru_type: 1 00000020:00100000:2.0:1775764773.546695:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1775764773.546701:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994efdf100 x1862022967001344/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764773.546721:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdf100 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967001344:192.168.202.146@tcp:36:rm.0 00000100:00100000:2.0:1775764773.546857:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102000 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.552587:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdf100 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967001344:192.168.202.146@tcp:36:rm.0 00000080:00200000:2.0:1775764773.552637:0:19896:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x287:0x0](ffff8f9952dba1c8) 00000080:00200000:2.0:1775764773.552714:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:2.0:1775764773.552723:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=WORDPRO, flags=0 00000080:00200008:2.0:1775764773.552733:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file WORDPRO:[0x200000401:0x2a9:0x0](ffff8f9952da6c08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764773.552775:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2a9:0x0](ffff8f9952da6c08),name=WORDPRO 00000002:00010000:2.0:1775764773.552799:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2a9:0x0]) in obj [0x200000401:0x26b:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764773.552812:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/681 (0 0) 00010000:00010000:2.0:1775764773.552859:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c737 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.552869:0:19896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c737 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.552878:0:19896:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c737 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764773.552900:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdc000 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967001472:192.168.202.146@tcp:101:rm.0 00000100:00100000:2.0:1775764773.552984:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102000 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.556131:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdc000 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967001472:192.168.202.146@tcp:101:rm.0 00010000:00010000:2.0:1775764773.556156:0:19896:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c737 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae07 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.556175:0:19896:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c737 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae07 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.556183:0:19896:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c737 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae07 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.556195:0:19896:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c737 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fae07 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.556206:0:19896:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c737 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae07 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.556219:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c737 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fae07 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.556227:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c737 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fae07 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.556235:0:19896:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c737 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fae07 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764773.556247:0:19896:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994efdc000 x1862022967001472/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/584 e 0 to 0 dl 1775764789 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764773.556261:0:19896:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c737 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae07 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.556273:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/681 (0 0) 00000080:00200000:2.0:1775764773.556280:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26b:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764773.556303:0:19896:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x2a9:0x0], updating i_size 7168 00000080:00010000:2.0:1775764773.556311:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2a9:0x0](ffff8f9952da6c08) for lock 0x36f9bdc9e421c737 00000080:00010000:2.0:1775764773.556319:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2a9:0x0](ffff8f9952da6c08) 00000080:00010000:2.0:1775764773.556322:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c737 from it ffffafbac1ec7cb8 00010000:00010000:2.0:1775764773.556325:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c737 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fae07 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.556334:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c737 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae07 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764773.556405:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:2.0:1775764773.556412:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=WORDPRO, flags=258 00000080:00200000:2.0:1775764773.556418:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2a9:0x0](ffff8f9952da6c08), inode mode 41c0 mask 44 00000080:00200008:2.0:1775764773.556426:0:19896:0:(file.c:1057:ll_file_open()) START file WORDPRO:[0x200000401:0x2a9:0x0](ffff8f9952da6c08), flags 704400 00000080:00200008:2.0:1775764773.556445:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file WORDPRO:[0x200000401:0x2a9:0x0](ffff8f9952da6c08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764773.556449:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2a9:0x0](ffff8f9952da6c08),name=WORDPRO 00000002:00010000:2.0:1775764773.556466:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2a9:0x0]) in obj [0x200000401:0x26b:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764773.556476:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c737 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae07 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764773.556485:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c737 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae07 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764773.556769:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26b:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764773.556775:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2a9:0x0](ffff8f9952da6c08) 00000080:00010000:2.0:1775764773.556778:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2a9:0x0](ffff8f9952da6c08) for lock 0x36f9bdc9e421c737 00000080:00010000:2.0:1775764773.556782:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c737 from it ffffafbac1ec7cd8 00010000:00010000:2.0:1775764773.556785:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c737 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fae07 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764773.556798:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c737 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae07 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764773.556821:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2a9:0x0](ffff8f9952da6c08) pos/size0/7168 32bit_api 0 00000002:00010000:2.0:1775764773.556834:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2a9:0x0]) in obj [0x200000401:0x2a9:0x0], intent: readdir flags 0 00010000:00010000:2.0:1775764773.556841:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c737 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae07 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764773.556848:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c737 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae07 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000100:00100000:2.0:1775764773.557409:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdc000 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967001600:192.168.202.146@tcp:37:rm.0 00000100:00100000:2.0:1775764773.557502:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102000 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.567822:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdc000 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967001600:192.168.202.146@tcp:37:rm.0 00010000:00010000:2.0:1775764773.568416:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c737 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fae07 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764773.568432:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c737 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae07 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764773.568483:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2a9:0x0](ffff8f9952da6c08) pos/size18446744073709551614/7168 32bit_api 0 00000080:00200008:3.0:1775764773.569110:0:19896:0:(file.c:447:ll_file_release()) START file WORDPRO:[0x200000401:0x2a9:0x0](ffff8f9952da6c08), flags 704000 00010000:00010000:3.0:1775764773.569141:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d8600/0x36f9bdc9e41ca8a8 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2a9:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ee113 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00000080:00200000:3.0:1775764773.569175:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:3.0:1775764773.569184:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=WORDPRO, flags=0 00000080:00200000:3.0:1775764773.569191:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 3 00000080:00200000:3.0:1775764773.569196:0:19896:0:(namei.c:2305:ll_rmdir()) VFS Op:name=WORDPRO, dir=[0x200000401:0x26b:0x0](ffff8f9952db00c8) 00010000:00010000:3.0:1775764773.569223:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c737 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fae07 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764773.569278:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970abe800/0x36f9bdc9e41ca8f5 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x11/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e4ee1ec expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:3.0:1775764773.569290:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d8600/0x36f9bdc9e41ca8a8 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2a9:0x0].0x0 bits 0x5/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e4ee113 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00000002:00100000:3.0:1775764773.569342:0:19896:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f9970f16300 x1862022950343424/t4294969621(4294969621) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764070 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000020:00100000:3.0:1775764773.569354:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:3.0:1775764773.569360:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970edc000 x1862022967001856/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:3.0:1775764773.569387:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967001856:192.168.202.146@tcp:35:rm.0 00000100:00100000:3.0:1775764773.569811:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413540 going to sleep for 11 seconds 00000100:00100000:0.0:1775764773.578144:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967001856:192.168.202.146@tcp:35:rm.0 00000002:00100000:0.0:1775764773.578178:0:19896:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f9970f16300 x1862022950343424/t4294969621(4294969621) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764070 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00010000:00010000:0.0:1775764773.578279:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c737 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fae07 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764773.578290:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970abe800/0x36f9bdc9e41ca8f5 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x11/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ee1ec expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:0.0:1775764773.578298:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d8600/0x36f9bdc9e41ca8a8 lrc: 1/0,0 mode: --/CR res: [0x200000401:0x2a9:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ee113 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00010000:00010000:0.0:1775764773.578304:0:19896:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:0.0:1775764773.578307:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c737 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fae07 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764773.578326:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970abe800/0x36f9bdc9e41ca8f5 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2a9:0x0].0x0 bits 0x11/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ee1ec expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:0.0:1775764773.578333:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d8600/0x36f9bdc9e41ca8a8 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x2a9:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ee113 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00000020:00100000:0.0:1775764773.578356:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1775764773.578363:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f0c8a80 x1862022967002112/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764773.578385:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c8a80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967002112:192.168.202.146@tcp:36:rm.0 00000100:00100000:0.0:1775764773.578474:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f180 going to sleep for 11 seconds 00000100:00100000:0.0:1775764773.589710:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c8a80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967002112:192.168.202.146@tcp:36:rm.0 00000080:00200000:0.0:1775764773.589805:0:19896:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2a9:0x0](ffff8f9952da6c08) 00000080:00200000:0.0:1775764773.589907:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764773.589927:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PM, flags=0 00000080:00200008:0.0:1775764773.589938:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file PM:[0x200000401:0x2b1:0x0](ffff8f9952d142c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764773.589943:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2b1:0x0](ffff8f9952d142c8),name=PM 00000002:00010000:0.0:1775764773.589966:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2b1:0x0]) in obj [0x200000401:0x26b:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764773.589978:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/689 (0 0) 00010000:00010000:0.0:1775764773.590029:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c73e lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.590038:0:19896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c73e lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.590052:0:19896:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c73e lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764773.590073:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c9180 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967002240:192.168.202.146@tcp:101:rm.0 00000100:00100000:0.0:1775764773.590158:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f180 going to sleep for 11 seconds 00000100:00100000:0.0:1775764773.595567:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c9180 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967002240:192.168.202.146@tcp:101:rm.0 00010000:00010000:0.0:1775764773.595589:0:19896:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c73e lrc: 4/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae2a expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.595624:0:19896:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c73e lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae2a expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.595633:0:19896:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c73e lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae2a expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.595645:0:19896:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c73e lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fae2a expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.595655:0:19896:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c73e lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae2a expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.595667:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c73e lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fae2a expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.595675:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c73e lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fae2a expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.595689:0:19896:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c73e lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fae2a expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764773.595699:0:19896:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994f0c9180 x1862022967002240/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/584 e 0 to 0 dl 1775764789 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1775764773.595712:0:19896:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c73e lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae2a expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.595721:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/689 (0 0) 00000080:00200000:0.0:1775764773.595728:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26b:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1775764773.595807:0:19896:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x2b1:0x0], updating i_size 7168 00000080:00010000:0.0:1775764773.595813:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2b1:0x0](ffff8f9952d142c8) for lock 0x36f9bdc9e421c73e 00000080:00010000:0.0:1775764773.595822:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2b1:0x0](ffff8f9952d142c8) 00000080:00010000:0.0:1775764773.595825:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c73e from it ffffafbac1ec7cb8 00010000:00010000:0.0:1775764773.595829:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c73e lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fae2a expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.595839:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c73e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae2a expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764773.595928:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764773.595936:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PM, flags=258 00000080:00200000:0.0:1775764773.595941:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2b1:0x0](ffff8f9952d142c8), inode mode 41c0 mask 44 00000080:00200008:0.0:1775764773.595949:0:19896:0:(file.c:1057:ll_file_open()) START file PM:[0x200000401:0x2b1:0x0](ffff8f9952d142c8), flags 704400 00000080:00200008:0.0:1775764773.595967:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file PM:[0x200000401:0x2b1:0x0](ffff8f9952d142c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764773.595973:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2b1:0x0](ffff8f9952d142c8),name=PM 00000002:00010000:0.0:1775764773.595990:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2b1:0x0]) in obj [0x200000401:0x26b:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764773.596000:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c73e lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae2a expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764773.596009:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c73e lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae2a expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764773.596016:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26b:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764773.596020:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2b1:0x0](ffff8f9952d142c8) 00000080:00010000:0.0:1775764773.596022:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2b1:0x0](ffff8f9952d142c8) for lock 0x36f9bdc9e421c73e 00000080:00010000:0.0:1775764773.596026:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c73e from it ffffafbac1ec7cd8 00010000:00010000:0.0:1775764773.596029:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c73e lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fae2a expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764773.596036:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c73e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae2a expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764773.596062:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2b1:0x0](ffff8f9952d142c8) pos/size0/7168 32bit_api 0 00000002:00010000:0.0:1775764773.596075:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2b1:0x0]) in obj [0x200000401:0x2b1:0x0], intent: readdir flags 0 00010000:00010000:0.0:1775764773.596081:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c73e lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae2a expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764773.596088:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c73e lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae2a expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000100:00100000:0.0:1775764773.596694:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c9180 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967002368:192.168.202.146@tcp:37:rm.0 00000100:00100000:0.0:1775764773.596830:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f180 going to sleep for 11 seconds 00000100:00100000:0.0:1775764773.612211:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c9180 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967002368:192.168.202.146@tcp:37:rm.0 00010000:00010000:0.0:1775764773.612910:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c73e lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fae2a expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764773.613090:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c73e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae2a expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1775764773.613140:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2b1:0x0](ffff8f9952d142c8) pos/size18446744073709551614/7168 32bit_api 0 00000080:00200008:0.0:1775764773.613160:0:19896:0:(file.c:447:ll_file_release()) START file PM:[0x200000401:0x2b1:0x0](ffff8f9952d142c8), flags 704000 00010000:00010000:0.0:1775764773.613185:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970abf400/0x36f9bdc9e41ca9ce lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2b1:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ee43f expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 62 lru_type: 1 00000080:00200000:0.0:1775764773.613217:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:0.0:1775764773.613227:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=PM, flags=0 00000080:00200000:0.0:1775764773.613233:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 3 00000080:00200000:0.0:1775764773.613239:0:19896:0:(namei.c:2305:ll_rmdir()) VFS Op:name=PM, dir=[0x200000401:0x26b:0x0](ffff8f9952db00c8) 00010000:00010000:0.0:1775764773.613263:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c73e lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fae2a expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764773.613333:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970abf800/0x36f9bdc9e41caa0d lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x11/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e4ee50a expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 0 00010000:00010000:0.0:1775764773.613347:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970abf400/0x36f9bdc9e41ca9ce lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2b1:0x0].0x0 bits 0x5/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e4ee43f expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 62 lru_type: 1 00000002:00100000:2.0:1775764773.623024:0:19896:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f994f2e5500 x1862022950354688/t4294969641(4294969641) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764071 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764773.623044:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764773.623051:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994efdc000 x1862022967002624/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764773.623079:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdc000 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967002624:192.168.202.146@tcp:35:rm.0 00000100:00100000:2.0:1775764773.623435:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102000 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.631088:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdc000 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967002624:192.168.202.146@tcp:35:rm.0 00000002:00100000:2.0:1775764773.631123:0:19896:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f994f2e5500 x1862022950354688/t4294969641(4294969641) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764071 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00010000:00010000:2.0:1775764773.631216:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c73e lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fae2a expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764773.631227:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970abf800/0x36f9bdc9e41caa0d lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x11/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ee50a expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 0 00010000:00010000:2.0:1775764773.631234:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970abf400/0x36f9bdc9e41ca9ce lrc: 1/0,0 mode: --/CR res: [0x200000401:0x2b1:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ee43f expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 62 lru_type: 1 00010000:00010000:2.0:1775764773.631240:0:19896:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:2.0:1775764773.631243:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd400/0x36f9bdc9e421c73e lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fae2a expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764773.631255:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970abf800/0x36f9bdc9e41caa0d lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2b1:0x0].0x0 bits 0x11/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ee50a expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 0 00010000:00010000:2.0:1775764773.631263:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970abf400/0x36f9bdc9e41ca9ce lrc: 0/0,0 mode: --/CR res: [0x200000401:0x2b1:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ee43f expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 62 lru_type: 1 00000020:00100000:2.0:1775764773.631283:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1775764773.631289:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994efdc000 x1862022967002880/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764773.631312:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdc000 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967002880:192.168.202.146@tcp:36:rm.0 00000100:00100000:2.0:1775764773.631403:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102000 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.650109:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdc000 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967002880:192.168.202.146@tcp:36:rm.0 00000080:00200000:2.0:1775764773.650167:0:19896:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2b1:0x0](ffff8f9952d142c8) 00000080:00200000:2.0:1775764773.650265:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:2.0:1775764773.650275:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=COREL, flags=0 00000080:00200008:2.0:1775764773.650286:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file COREL:[0x200000401:0x292:0x0](ffff8f9952d89148), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764773.650293:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x292:0x0](ffff8f9952d89148),name=COREL 00000002:00010000:2.0:1775764773.650316:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x292:0x0]) in obj [0x200000401:0x26b:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764773.650328:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/658 (0 0) 00010000:00010000:2.0:1775764773.650396:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c745 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.650406:0:19896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c745 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.650415:0:19896:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c745 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764773.650437:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efded80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967003008:192.168.202.146@tcp:101:rm.0 00000100:00100000:2.0:1775764773.650536:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102000 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.657129:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efded80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967003008:192.168.202.146@tcp:101:rm.0 00010000:00010000:2.0:1775764773.657155:0:19896:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c745 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae4d expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.657172:0:19896:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c745 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x292:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae4d expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.657181:0:19896:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c745 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae4d expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.657193:0:19896:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c745 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fae4d expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.657204:0:19896:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c745 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae4d expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.657216:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c745 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fae4d expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.657224:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c745 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fae4d expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.657232:0:19896:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c745 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fae4d expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764773.657243:0:19896:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994efded80 x1862022967003008/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/584 e 0 to 0 dl 1775764789 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764773.657254:0:19896:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c745 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae4d expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.657263:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/658 (0 0) 00000080:00200000:2.0:1775764773.657269:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26b:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764773.657289:0:19896:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x292:0x0], updating i_size 7168 00000080:00010000:2.0:1775764773.657295:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x292:0x0](ffff8f9952d89148) for lock 0x36f9bdc9e421c745 00000080:00010000:2.0:1775764773.657305:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x292:0x0](ffff8f9952d89148) 00000080:00010000:2.0:1775764773.657307:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c745 from it ffffafbac1ec7cb8 00010000:00010000:2.0:1775764773.657311:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c745 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fae4d expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.657318:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c745 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae4d expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764773.657845:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:2.0:1775764773.657856:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=COREL, flags=258 00000080:00200000:2.0:1775764773.657864:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x292:0x0](ffff8f9952d89148), inode mode 41c0 mask 44 00000080:00200008:2.0:1775764773.657872:0:19896:0:(file.c:1057:ll_file_open()) START file COREL:[0x200000401:0x292:0x0](ffff8f9952d89148), flags 704400 00000080:00200008:2.0:1775764773.657891:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file COREL:[0x200000401:0x292:0x0](ffff8f9952d89148), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764773.657896:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x292:0x0](ffff8f9952d89148),name=COREL 00000002:00010000:2.0:1775764773.657925:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x292:0x0]) in obj [0x200000401:0x26b:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764773.657937:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c745 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae4d expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764773.657948:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c745 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae4d expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764773.657956:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26b:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764773.657960:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x292:0x0](ffff8f9952d89148) 00000080:00010000:2.0:1775764773.657964:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x292:0x0](ffff8f9952d89148) for lock 0x36f9bdc9e421c745 00000080:00010000:2.0:1775764773.657968:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c745 from it ffffafbac1ec7cd8 00010000:00010000:2.0:1775764773.657971:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c745 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fae4d expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764773.657984:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c745 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae4d expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764773.658011:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x292:0x0](ffff8f9952d89148) pos/size0/7168 32bit_api 0 00000002:00010000:2.0:1775764773.658022:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x292:0x0]) in obj [0x200000401:0x292:0x0], intent: readdir flags 0 00010000:00010000:2.0:1775764773.658028:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c745 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae4d expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764773.658035:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c745 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae4d expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000100:00100000:2.0:1775764773.659010:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efded80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967003136:192.168.202.146@tcp:37:rm.0 00000100:00100000:2.0:1775764773.659111:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102000 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.667548:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efded80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967003136:192.168.202.146@tcp:37:rm.0 00010000:00010000:2.0:1775764773.668258:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c745 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fae4d expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764773.668273:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c745 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae4d expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764773.668322:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x292:0x0](ffff8f9952d89148) pos/size18446744073709551614/7168 32bit_api 0 00000080:00200008:2.0:1775764773.668341:0:19896:0:(file.c:447:ll_file_release()) START file COREL:[0x200000401:0x292:0x0](ffff8f9952d89148), flags 704000 00010000:00010000:2.0:1775764773.668366:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971079200/0x36f9bdc9e41ca56e lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x292:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ed8fb expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00000080:00200000:2.0:1775764773.668397:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:2.0:1775764773.668408:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=COREL, flags=0 00000080:00200000:2.0:1775764773.668414:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 3 00000080:00200000:2.0:1775764773.668419:0:19896:0:(namei.c:2305:ll_rmdir()) VFS Op:name=COREL, dir=[0x200000401:0x26b:0x0](ffff8f9952db00c8) 00010000:00010000:2.0:1775764773.668442:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c745 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fae4d expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764773.668511:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970fd6a00/0x36f9bdc9e41ca5ad lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x292:0x0].0x0 bits 0x11/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e4ed9b1 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764773.668524:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971079200/0x36f9bdc9e41ca56e lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x292:0x0].0x0 bits 0x5/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e4ed8fb expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00000002:00100000:2.0:1775764773.668570:0:19896:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f9970c02680 x1862022950312704/t4294969571(4294969571) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764069 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764773.668581:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764773.668586:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994efded80 x1862022967003392/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764773.668619:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efded80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967003392:192.168.202.146@tcp:35:rm.0 00000100:00100000:2.0:1775764773.668706:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102000 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.672438:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efded80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967003392:192.168.202.146@tcp:35:rm.0 00000002:00100000:2.0:1775764773.672471:0:19896:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f9970c02680 x1862022950312704/t4294969571(4294969571) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764069 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00010000:00010000:2.0:1775764773.672724:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c745 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fae4d expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764773.672736:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970fd6a00/0x36f9bdc9e41ca5ad lrc: 1/0,0 mode: --/PR res: [0x200000401:0x292:0x0].0x0 bits 0x11/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ed9b1 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764773.672804:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971079200/0x36f9bdc9e41ca56e lrc: 1/0,0 mode: --/CR res: [0x200000401:0x292:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ed8fb expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00010000:00010000:2.0:1775764773.672810:0:19896:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:2.0:1775764773.672813:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c745 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x292:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fae4d expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764773.672823:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970fd6a00/0x36f9bdc9e41ca5ad lrc: 0/0,0 mode: --/PR res: [0x200000401:0x292:0x0].0x0 bits 0x11/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ed9b1 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764773.672830:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971079200/0x36f9bdc9e41ca56e lrc: 0/0,0 mode: --/CR res: [0x200000401:0x292:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ed8fb expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00000020:00100000:2.0:1775764773.672849:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1775764773.672854:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994efded80 x1862022967003648/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764773.672877:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efded80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967003648:192.168.202.146@tcp:36:rm.0 00000100:00100000:2.0:1775764773.673078:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102000 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.677768:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efded80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967003648:192.168.202.146@tcp:36:rm.0 00000080:00200000:2.0:1775764773.677811:0:19896:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x292:0x0](ffff8f9952d89148) 00000080:00200000:2.0:1775764773.677900:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:2.0:1775764773.677909:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=ACCESS, flags=0 00000080:00200008:2.0:1775764773.677925:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file ACCESS:[0x200000401:0x2e1:0x0](ffff8f9952d91148), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764773.677930:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2e1:0x0](ffff8f9952d91148),name=ACCESS 00000002:00010000:2.0:1775764773.677950:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2e1:0x0]) in obj [0x200000401:0x26b:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764773.677962:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/737 (0 0) 00010000:00010000:2.0:1775764773.678012:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c74c lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.678021:0:19896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c74c lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.678031:0:19896:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c74c lrc: 3/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764773.678049:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdca80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967003776:192.168.202.146@tcp:101:rm.0 00000100:00100000:2.0:1775764773.679214:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102000 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.683211:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdca80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967003776:192.168.202.146@tcp:101:rm.0 00010000:00010000:2.0:1775764773.683239:0:19896:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c74c lrc: 4/1,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae70 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.683256:0:19896:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c74c lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae70 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.683264:0:19896:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c74c lrc: 4/1,0 mode: --/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae70 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.683278:0:19896:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c74c lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fae70 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.683290:0:19896:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c74c lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae70 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.683303:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c74c lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fae70 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.683313:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c74c lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fae70 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.683321:0:19896:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c74c lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fae70 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764773.683332:0:19896:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994efdca80 x1862022967003776/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/584 e 0 to 0 dl 1775764789 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764773.683344:0:19896:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c74c lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae70 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.683354:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/737 (0 0) 00000080:00200000:2.0:1775764773.683434:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26b:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764773.683456:0:19896:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x2e1:0x0], updating i_size 7168 00000080:00010000:2.0:1775764773.683462:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2e1:0x0](ffff8f9952d91148) for lock 0x36f9bdc9e421c74c 00000080:00010000:2.0:1775764773.683470:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2e1:0x0](ffff8f9952d91148) 00000080:00010000:2.0:1775764773.683474:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c74c from it ffffafbac1ec7cb8 00010000:00010000:2.0:1775764773.683477:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c74c lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fae70 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.683486:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c74c lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae70 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764773.683566:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:2.0:1775764773.683575:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=ACCESS, flags=258 00000080:00200000:2.0:1775764773.683581:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x2e1:0x0](ffff8f9952d91148), inode mode 41c0 mask 44 00000080:00200008:2.0:1775764773.683590:0:19896:0:(file.c:1057:ll_file_open()) START file ACCESS:[0x200000401:0x2e1:0x0](ffff8f9952d91148), flags 704400 00000080:00200008:2.0:1775764773.684113:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file ACCESS:[0x200000401:0x2e1:0x0](ffff8f9952d91148), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764773.684120:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x2e1:0x0](ffff8f9952d91148),name=ACCESS 00000002:00010000:2.0:1775764773.684140:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2e1:0x0]) in obj [0x200000401:0x26b:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764773.684150:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c74c lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae70 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764773.684161:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c74c lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae70 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764773.684168:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x26b:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764773.684173:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x2e1:0x0](ffff8f9952d91148) 00000080:00010000:2.0:1775764773.684176:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x2e1:0x0](ffff8f9952d91148) for lock 0x36f9bdc9e421c74c 00000080:00010000:2.0:1775764773.684180:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c74c from it ffffafbac1ec7cd8 00010000:00010000:2.0:1775764773.684183:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c74c lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fae70 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764773.684190:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c74c lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae70 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764773.684214:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2e1:0x0](ffff8f9952d91148) pos/size0/7168 32bit_api 0 00000002:00010000:2.0:1775764773.684225:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x2e1:0x0]) in obj [0x200000401:0x2e1:0x0], intent: readdir flags 0 00010000:00010000:2.0:1775764773.684231:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c74c lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae70 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764773.684239:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c74c lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae70 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000100:00100000:2.0:1775764773.685251:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdca80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967003904:192.168.202.146@tcp:37:rm.0 00000100:00100000:2.0:1775764773.685443:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102000 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.693891:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdca80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967003904:192.168.202.146@tcp:37:rm.0 00010000:00010000:2.0:1775764773.694681:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c74c lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fae70 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764773.694700:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c74c lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fae70 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764773.694778:0:19896:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x2e1:0x0](ffff8f9952d91148) pos/size18446744073709551614/7168 32bit_api 0 00000080:00200008:2.0:1775764773.694797:0:19896:0:(file.c:447:ll_file_release()) START file ACCESS:[0x200000401:0x2e1:0x0](ffff8f9952d91148), flags 704000 00010000:00010000:2.0:1775764773.694823:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9000/0x36f9bdc9e41cb073 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2e1:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ef4ed expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00200000:2.0:1775764773.694858:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 1 00000080:00200000:2.0:1775764773.694868:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=ACCESS, flags=0 00000080:00200000:2.0:1775764773.694875:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8), inode mode 41c0 mask 3 00000080:00200000:2.0:1775764773.694882:0:19896:0:(namei.c:2305:ll_rmdir()) VFS Op:name=ACCESS, dir=[0x200000401:0x26b:0x0](ffff8f9952db00c8) 00010000:00010000:2.0:1775764773.694907:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c74c lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fae70 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764773.694978:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a44a00/0x36f9bdc9e41cf6e3 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x11/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e4fd1cf expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764773.694991:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9000/0x36f9bdc9e41cb073 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x2e1:0x0].0x0 bits 0x5/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e4ef4ed expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000002:00100000:2.0:1775764773.695719:0:19896:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f9970c01180 x1862022950420096/t4294969747(4294969747) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764075 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764773.695732:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764773.695780:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994efdca80 x1862022967004160/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764773.695807:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdca80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967004160:192.168.202.146@tcp:35:rm.0 00000100:00100000:2.0:1775764773.695913:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102000 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.699814:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdca80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967004160:192.168.202.146@tcp:35:rm.0 00000002:00100000:2.0:1775764773.699847:0:19896:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f9970c01180 x1862022950420096/t4294969747(4294969747) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764075 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00010000:00010000:2.0:1775764773.699937:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c74c lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fae70 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764773.699947:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a44a00/0x36f9bdc9e41cf6e3 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x11/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4fd1cf expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764773.699953:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9000/0x36f9bdc9e41cb073 lrc: 1/0,0 mode: --/CR res: [0x200000401:0x2e1:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ef4ed expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:2.0:1775764773.699959:0:19896:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:2.0:1775764773.699962:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c74c lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fae70 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764773.699973:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970a44a00/0x36f9bdc9e41cf6e3 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x2e1:0x0].0x0 bits 0x11/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4fd1cf expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764773.699986:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99710c9000/0x36f9bdc9e41cb073 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x2e1:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ef4ed expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000020:00100000:2.0:1775764773.700005:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1775764773.700011:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994efdca80 x1862022967004416/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764773.700032:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdca80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967004416:192.168.202.146@tcp:36:rm.0 00000100:00100000:2.0:1775764773.700175:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102000 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.706911:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdca80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967004416:192.168.202.146@tcp:36:rm.0 00000080:00200000:2.0:1775764773.706967:0:19896:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x2e1:0x0](ffff8f9952d91148) 00000080:00200008:2.0:1775764773.707191:0:19896:0:(file.c:447:ll_file_release()) START file ~dmtmp:[0x200000401:0x26b:0x0](ffff8f9952db00c8), flags 704000 00010000:00010000:2.0:1775764773.707217:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9940030600/0x36f9bdc9e41ca521 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e4ed7e3 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764773.707255:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 1 00000080:00200000:2.0:1775764773.707268:0:19896:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=~dmtmp, dir=[0x200000401:0x269:0x0](ffff8f9952db63c8), flags=0 00000080:00200000:2.0:1775764773.707273:0:19896:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=~dmtmp, dir=[0x200000401:0x269:0x0](ffff8f9952db63c8), intent=getattr 00000002:00010000:2.0:1775764773.707295:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ~dmtmp,[0x0:0x0:0x0]) in obj [0x200000401:0x269:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764773.707508:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c753 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x269:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.707518:0:19896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c753 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x269:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.707528:0:19896:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c753 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x269:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764773.707549:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdc700 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967004544:192.168.202.146@tcp:101:rm.0 00000100:00100000:2.0:1775764773.707838:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102000 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.717680:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdc700 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967004544:192.168.202.146@tcp:101:rm.0 00010000:00010000:2.0:1775764773.717714:0:19896:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c753 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x269:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faea1 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.717733:0:19896:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c753 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faea1 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.717825:0:19896:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c753 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faea1 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.717836:0:19896:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c753 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5faea1 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.717847:0:19896:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c753 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faea1 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.717859:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c753 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5faea1 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.717866:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c753 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5faea1 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.717873:0:19896:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c753 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5faea1 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764773.717889:0:19896:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994efdc700 x1862022967004544/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/584 e 0 to 0 dl 1775764789 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764773.717902:0:19896:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c753 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faea1 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.717911:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/619 (0 0) 00000080:00200000:2.0:1775764773.718117:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x269:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764773.718144:0:19896:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x26b:0x0], updating i_size 7168 00000080:00200000:2.0:1775764773.718148:0:19896:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x26b:0x0](ffff8f9952db00c8): rc = 0 00000080:00010000:2.0:1775764773.718154:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x26b:0x0](ffff8f9952db00c8) for lock 0x36f9bdc9e421c753 00000080:00010000:2.0:1775764773.718165:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x26b:0x0](ffff8f9952db00c8) 00000080:00010000:2.0:1775764773.718168:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c753 from it ffffafbac1ec7e00 00010000:00010000:2.0:1775764773.718171:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c753 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5faea1 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.718179:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c753 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faea1 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764773.718218:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8), inode mode 41c0 mask 3 00000080:00200000:2.0:1775764773.718227:0:19896:0:(namei.c:2305:ll_rmdir()) VFS Op:name=~dmtmp, dir=[0x200000401:0x269:0x0](ffff8f9952db63c8) 00010000:00010000:2.0:1775764773.718254:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c6ff lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x269:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fad27 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764773.718272:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b9600/0x36f9bdc9e421c205 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x269:0x0].0x0 bits 0x11/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5f8a58 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764773.718503:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c753 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5faea1 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764773.718518:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9940030600/0x36f9bdc9e41ca521 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e4ed7e3 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00000002:00100000:2.0:1775764773.718608:0:19896:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f9970edd500 x1862022950309248/t4294969563(4294969563) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764069 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764773.718618:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764773.718623:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994efdc700 x1862022967004800/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764773.718647:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdc700 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967004800:192.168.202.146@tcp:35:rm.0 00000100:00100000:2.0:1775764773.718834:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102000 going to sleep for 11 seconds 00000100:00100000:0.0:1775764773.723260:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdc700 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967004800:192.168.202.146@tcp:35:rm.0 00000002:00100000:0.0:1775764773.723293:0:19896:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f9970edd500 x1862022950309248/t4294969563(4294969563) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764069 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dbench.0' uid:0 gid:0 projid:0 00010000:00010000:0.0:1775764773.723899:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c753 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5faea1 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764773.723910:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9940030600/0x36f9bdc9e41ca521 lrc: 1/0,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ed7e3 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764773.724034:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c6ff lrc: 1/0,0 mode: --/PR res: [0x200000401:0x269:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fad27 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764773.724122:0:19896:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:0.0:1775764773.724125:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c753 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x26b:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5faea1 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764773.724137:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9940030600/0x36f9bdc9e41ca521 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x26b:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e4ed7e3 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764773.724150:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd200/0x36f9bdc9e421c6ff lrc: 0/0,0 mode: --/PR res: [0x200000401:0x269:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fad27 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000020:00100000:0.0:1775764773.724167:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1775764773.724176:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f0c9180 x1862022967005056/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764773.724198:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c9180 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967005056:192.168.202.146@tcp:36:rm.0 00000100:00100000:0.0:1775764773.724289:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f180 going to sleep for 11 seconds 00000100:00100000:0.0:1775764773.734463:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c9180 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967005056:192.168.202.146@tcp:36:rm.0 00000080:00200000:0.0:1775764773.734559:0:19896:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x26b:0x0](ffff8f9952db00c8) 00000080:00200008:1.0:1775764773.735187:0:19896:0:(file.c:447:ll_file_release()) START file client1:[0x200000401:0x269:0x0](ffff8f9952db63c8), flags 704000 00010000:00010000:1.0:1775764773.735218:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c70d lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x269:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fad43 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764773.735255:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764773.735266:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client1, flags=0 00000080:00200000:1.0:1775764773.735272:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88), inode mode 41ed mask 3 00000080:00200000:1.0:1775764773.735276:0:19896:0:(namei.c:2305:ll_rmdir()) VFS Op:name=client1, dir=[0x200000401:0x268:0x0](ffff8f9952e63a88) 00010000:00010000:1.0:1775764773.735302:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c70d lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x269:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fad43 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000002:00100000:1.0:1775764773.735371:0:19896:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f994f2e7b80 x1862022966997376/t4294997559(4294997559) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764789 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'rm.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1775764773.735383:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764773.735388:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e7480 x1862022967005312/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764773.735411:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7480 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967005312:192.168.202.146@tcp:35:rm.0 00000100:00100000:1.0:1775764773.735510:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f997130d900 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.748588:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7480 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967005312:192.168.202.146@tcp:35:rm.0 00000002:00100000:2.0:1775764773.748706:0:19896:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f994f2e7b80 x1862022966997376/t4294997559(4294997559) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764789 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'rm.0' uid:0 gid:0 projid:0 00010000:00010000:2.0:1775764773.748868:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b9600/0x36f9bdc9e421c205 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x269:0x0].0x0 bits 0x11/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5f8a58 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764773.749365:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c70d lrc: 1/0,0 mode: --/CR res: [0x200000401:0x269:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fad43 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764773.749376:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b9600/0x36f9bdc9e421c205 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x269:0x0].0x0 bits 0x11/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8a58 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764773.749388:0:19896:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:2.0:1775764773.749391:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c70d lrc: 0/0,0 mode: --/CR res: [0x200000401:0x269:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fad43 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764773.749405:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99668b9600/0x36f9bdc9e421c205 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x269:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5f8a58 expref: -99 pid: 19724 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000020:00100000:2.0:1775764773.749429:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1775764773.749437:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994efdd500 x1862022967005568/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 488/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764773.749462:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdd500 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967005568:192.168.202.146@tcp:36:rm.0 00000100:00100000:2.0:1775764773.749573:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102c00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.755881:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdd500 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967005568:192.168.202.146@tcp:36:rm.0 00000080:00200000:2.0:1775764773.755936:0:19896:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x269:0x0](ffff8f9952db63c8) 00000080:00200008:2.0:1775764773.756018:0:19896:0:(file.c:447:ll_file_release()) START file clients:[0x200000401:0x268:0x0](ffff8f9952e63a88), flags 704000 00010000:00010000:2.0:1775764773.756052:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb400/0x36f9bdc9e421c69d lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x268:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa8d5 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764773.756087:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:2.0:1775764773.756103:0:19896:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=clients, dir=[0x200000401:0x261:0x0](ffff8f9952e61988), flags=0 00000080:00200000:2.0:1775764773.756107:0:19896:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=clients, dir=[0x200000401:0x261:0x0](ffff8f9952e61988), intent=getattr 00000002:00010000:2.0:1775764773.756281:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: clients,[0x0:0x0:0x0]) in obj [0x200000401:0x261:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764773.756399:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c75a lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.756408:0:19896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c75a lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.756418:0:19896:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c75a lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764773.756438:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdc380 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967005696:192.168.202.146@tcp:101:rm.0 00000100:00100000:2.0:1775764773.756518:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102c00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.763678:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdc380 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967005696:192.168.202.146@tcp:101:rm.0 00010000:00010000:2.0:1775764773.763705:0:19896:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c75a lrc: 4/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faee7 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.763722:0:19896:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c75a lrc: 4/1,0 mode: --/PR res: [0x200000401:0x268:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faee7 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.763731:0:19896:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c75a lrc: 4/1,0 mode: --/PR res: [0x200000401:0x268:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faee7 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.763796:0:19896:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c75a lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x268:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5faee7 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.763808:0:19896:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c75a lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x268:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faee7 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.763820:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c75a lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x268:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5faee7 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.763829:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c75a lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x268:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5faee7 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.763841:0:19896:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c75a lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x268:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5faee7 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764773.763853:0:19896:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994efdc380 x1862022967005696/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/584 e 0 to 0 dl 1775764789 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764773.763865:0:19896:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c75a lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x268:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faee7 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.763874:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/616 (0 0) 00000080:00200000:2.0:1775764773.763881:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x261:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764773.763904:0:19896:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x268:0x0], updating i_size 7168 00000080:00200000:2.0:1775764773.763907:0:19896:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x268:0x0](ffff8f9952e63a88): rc = 0 00000080:00010000:2.0:1775764773.763913:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x268:0x0](ffff8f9952e63a88) for lock 0x36f9bdc9e421c75a 00000080:00010000:2.0:1775764773.763935:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x268:0x0](ffff8f9952e63a88) 00000080:00010000:2.0:1775764773.763939:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c75a from it ffffafbac1ec7e00 00010000:00010000:2.0:1775764773.763942:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c75a lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x268:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5faee7 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.763951:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c75a lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x268:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faee7 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764773.763993:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 3 00000080:00200000:2.0:1775764773.764002:0:19896:0:(namei.c:2305:ll_rmdir()) VFS Op:name=clients, dir=[0x200000401:0x261:0x0](ffff8f9952e61988) 00010000:00010000:2.0:1775764773.764027:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 33 lru_type: 1 00010000:00010000:2.0:1775764773.764305:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c75a lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x268:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5faee7 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764773.764319:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb400/0x36f9bdc9e421c69d lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x268:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fa8d5 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000002:00100000:2.0:1775764773.764360:0:19896:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f994efde680 x1862022966975232/t4294997531(4294997531) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764788 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'rm.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764773.764371:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764773.764376:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994efdc380 x1862022967005952/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764773.764399:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdc380 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967005952:192.168.202.146@tcp:35:rm.0 00000100:00100000:2.0:1775764773.764492:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102c00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.768699:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdc380 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967005952:192.168.202.146@tcp:35:rm.0 00000002:00100000:2.0:1775764773.768733:0:19896:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f994efde680 x1862022966975232/t4294997531(4294997531) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764788 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'rm.0' uid:0 gid:0 projid:0 00010000:00010000:2.0:1775764773.768868:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c75a lrc: 1/0,0 mode: --/PR res: [0x200000401:0x268:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5faee7 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764773.768878:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb400/0x36f9bdc9e421c69d lrc: 1/0,0 mode: --/CR res: [0x200000401:0x268:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fa8d5 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764773.768886:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 33 lru_type: 1 00010000:00010000:2.0:1775764773.768896:0:19896:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:2.0:1775764773.768899:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c75a lrc: 0/0,0 mode: --/PR res: [0x200000401:0x268:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5faee7 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764773.768910:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb400/0x36f9bdc9e421c69d lrc: 0/0,0 mode: --/CR res: [0x200000401:0x268:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fa8d5 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764773.768934:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c5d9 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fa690 expref: -99 pid: 19893 timeout: 0 lvb_type: 0 lru_score: 33 lru_type: 1 00000020:00100000:2.0:1775764773.768951:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1775764773.768958:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994efdc380 x1862022967006208/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764773.768980:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdc380 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967006208:192.168.202.146@tcp:36:rm.0 00000100:00100000:2.0:1775764773.769072:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102c00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.776835:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdc380 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967006208:192.168.202.146@tcp:36:rm.0 00000080:00200000:2.0:1775764773.776878:0:19896:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x268:0x0](ffff8f9952e63a88) 00000080:00200000:2.0:1775764773.776969:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:2.0:1775764773.776988:0:19896:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=client.txt, dir=[0x200000401:0x261:0x0](ffff8f9952e61988), flags=0 00000080:00200000:2.0:1775764773.776993:0:19896:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=client.txt, dir=[0x200000401:0x261:0x0](ffff8f9952e61988), intent=getattr 00000002:00010000:2.0:1775764773.777013:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: client.txt,[0x0:0x0:0x0]) in obj [0x200000401:0x261:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764773.777075:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c761 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.777085:0:19896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c761 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.777095:0:19896:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c761 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764773.777113:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efde680 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967006336:192.168.202.146@tcp:101:rm.0 00000100:00100000:2.0:1775764773.777507:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102c00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.784190:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efde680 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967006336:192.168.202.146@tcp:101:rm.0 00010000:00010000:2.0:1775764773.784216:0:19896:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c761 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faf18 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.784242:0:19896:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c761 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x262:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faf18 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.784249:0:19896:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c761 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x262:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faf18 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.784260:0:19896:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c761 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x262:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5faf18 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.784270:0:19896:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c761 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x262:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faf18 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.784282:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c761 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x262:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5faf18 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.784290:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c761 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x262:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5faf18 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.784302:0:19896:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c761 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x262:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5faf18 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764773.784324:0:19896:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994efde680 x1862022967006336/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/976 e 0 to 0 dl 1775764789 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764773.784335:0:19896:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 392 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c761 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x262:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faf18 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764773.784349:0:19896:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c761 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x262:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faf18 expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.784368:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/610 (0 0) 00000080:00200000:2.0:1775764773.784375:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x261:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764773.784465:0:19896:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x262:0x0](ffff8f9952db1148): rc = 0 00000080:00200000:2.0:1775764773.784489:0:19896:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x262:0x0]: layout version change: 5 -> 5 00000080:00010000:2.0:1775764773.784500:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x262:0x0](ffff8f9952db1148) for lock 0x36f9bdc9e421c761 00000080:00010000:2.0:1775764773.784511:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x262:0x0](ffff8f9952db1148) 00000080:00010000:2.0:1775764773.784515:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c761 from it ffffafbac1ec7b20 00010000:00010000:2.0:1775764773.784518:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c761 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x262:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5faf18 expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.784527:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c761 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x262:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5faf18 expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200008:2.0:1775764773.784571:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file client.txt:[0x200000401:0x262:0x0](ffff8f9952db1148), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764773.784579:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x262:0x0](ffff8f9952db1148),name=client.txt 00000002:00010000:2.0:1775764773.784621:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x262:0x0]) in obj [0x200000401:0x261:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764773.784631:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c761 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x262:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5faf18 expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764773.784642:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c761 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x262:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5faf18 expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764773.784649:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x261:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764773.784654:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x262:0x0](ffff8f9952db1148) 00000080:00010000:2.0:1775764773.784657:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x262:0x0](ffff8f9952db1148) for lock 0x36f9bdc9e421c761 00000080:00010000:2.0:1775764773.784660:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c761 from it ffffafbac1ec7cb8 00010000:00010000:2.0:1775764773.784664:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c761 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x262:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5faf18 expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764773.784670:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c761 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x262:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5faf18 expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764773.784698:0:19896:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x262:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1775764773.784707:0:19896:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x262:0x0] 00000008:00010000:2.0:1775764773.784790:0:19896:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764773.784794:0:19896:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f9966b03630, osc lock ffff8f994f1a2850, flags 0x400001002 00000008:00010000:2.0:1775764773.784813:0:19896:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764773.784815:0:19896:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f9966b03670, osc lock ffff8f994f1a24c0, flags 0x400001002 00000008:00010000:2.0:1775764773.784830:0:19896:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764773.784833:0:19896:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f9966b036b0, osc lock ffff8f994f1a3560, flags 0x400001002 00000008:00010000:2.0:1775764773.784849:0:19896:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764773.784851:0:19896:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f9966b036f0, osc lock ffff8f994f1a2980, flags 0x400001002 00000008:00010000:2.0:1775764773.784863:0:19896:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764773.784865:0:19896:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f9966b03730, osc lock ffff8f994f1a37c0, flags 0x400001002 00010000:00010000:2.0:1775764773.784906:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ae8788d7/0x36f9bdc9e421c768 lrc: 3/1,0 mode: --/PR res: [0x240000403:0x2e:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.784917:0:19896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ae8788d7/0x36f9bdc9e421c768 lrc: 3/1,0 mode: --/PR res: [0x240000403:0x2e:0x0].0x0 rrc: 2 type: EXT [0->2097151] (req 0->2097151) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764773.784994:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efde680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967006464:192.168.202.146@tcp:101:rm.0 00010000:00010000:2.0:1775764773.785188:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005bcdc680/0x36f9bdc9e421c76f lrc: 3/1,0 mode: --/PR res: [0x280000403:0x2f:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.785196:0:19896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005bcdc680/0x36f9bdc9e421c76f lrc: 3/1,0 mode: --/PR res: [0x280000403:0x2f:0x0].0x0 rrc: 2 type: EXT [2097152->8388607] (req 2097152->8388607) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764773.789789:0:2381:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1775764773.789818:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967006592:192.168.202.146@tcp:101:rm.0 00010000:00010000:2.0:1775764773.791151:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000062300bdc/0x36f9bdc9e41c9bab lrc: 2/1,0 mode: PR/PR res: [0x240000403:0x2f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e4ebe4b expref: -99 pid: 19698 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:0.0:1775764773.791509:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 8388607) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000062300bdc/0x36f9bdc9e41c9bab lrc: 2/1,0 mode: PR/PR res: [0x240000403:0x2f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e4ebe4b expref: -99 pid: 19698 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00000020:00200000:0.0:1775764773.791527:0:19896:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00010000:00010000:0.0:1775764773.791531:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000062300bdc/0x36f9bdc9e41c9bab lrc: 5/1,0 mode: PR/PR res: [0x240000403:0x2f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8388607) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e4ebe4b expref: -99 pid: 19698 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:0.0:1775764773.791541:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000062300bdc/0x36f9bdc9e41c9bab lrc: 4/0,0 mode: PR/PR res: [0x240000403:0x2f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e4ebe4b expref: -99 pid: 19698 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:0.0:1775764773.791633:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c776 lrc: 3/1,0 mode: --/PR res: [0x240000403:0x30:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.791642:0:19896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c776 lrc: 3/1,0 mode: --/PR res: [0x240000403:0x30:0x0].0x0 rrc: 2 type: EXT [8388608->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764773.791688:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1775764773.791704:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967006720:192.168.202.146@tcp:101:rm.0 00010000:00010000:0.0:1775764773.791819:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005b4933ab/0x36f9bdc9e421c77d lrc: 3/1,0 mode: --/PR res: [0x280000403:0x30:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.791828:0:19896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005b4933ab/0x36f9bdc9e421c77d lrc: 3/1,0 mode: --/PR res: [0x280000403:0x30:0x0].0x0 rrc: 2 type: EXT [8388608->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:0.0:1775764773.791843:0:19896:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000100:00100000:0.0:1775764773.791856:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967006848:192.168.202.146@tcp:101:rm.0 00010000:00010000:3.0:1775764773.801581:0:2381:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ae8788d7/0x36f9bdc9e421c768 lrc: 6/2,0 mode: --/PR res: [0x240000403:0x2e:0x0].0x0 rrc: 2 type: EXT [0->2097151] (req 0->2097151) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.802014:0:2381:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ae8788d7/0x36f9bdc9e421c768 lrc: 6/2,0 mode: --/PR res: [0x240000403:0x2e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faf1f expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.802031:0:2381:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ae8788d7/0x36f9bdc9e421c768 lrc: 6/2,0 mode: PR/PR res: [0x240000403:0x2e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5faf1f expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764773.802240:0:2379:0:(client.c:3082:ptlrpc_free_committed()) lustre-OST0001-osc-ffff8f99523ee800: committing for last_committed 4294972285 gen 6 00010000:00010000:1.0:1775764773.802271:0:2379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005b4933ab/0x36f9bdc9e421c77d lrc: 6/2,0 mode: --/PR res: [0x280000403:0x30:0x0].0x0 rrc: 2 type: EXT [8388608->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764773.802284:0:2379:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005b4933ab/0x36f9bdc9e421c77d lrc: 6/2,0 mode: --/PR res: [0x280000403:0x30:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faf34 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764773.802293:0:2379:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005b4933ab/0x36f9bdc9e421c77d lrc: 6/2,0 mode: PR/PR res: [0x280000403:0x30:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5faf34 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764773.802303:0:2379:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005b4933ab/0x36f9bdc9e421c77d lrc: 6/2,0 mode: PR/PR res: [0x280000403:0x30:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faf34 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.802460:0:2381:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ae8788d7/0x36f9bdc9e421c768 lrc: 6/2,0 mode: PR/PR res: [0x240000403:0x2e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faf1f expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:3.0:1775764773.802476:0:2381:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 2097152, setting rss=2097152; kms=2097152, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ae8788d7/0x36f9bdc9e421c768 lrc: 5/2,0 mode: PR/PR res: [0x240000403:0x2e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5faf1f expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:3.0:1775764773.802489:0:2381:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00010000:00010000:3.0:1775764773.802493:0:2381:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ae8788d7/0x36f9bdc9e421c768 lrc: 6/2,0 mode: PR/PR res: [0x240000403:0x2e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5faf1f expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.802502:0:2381:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ae8788d7/0x36f9bdc9e421c768 lrc: 5/1,0 mode: PR/PR res: [0x240000403:0x2e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5faf1f expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.802510:0:2381:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ae8788d7/0x36f9bdc9e421c768 lrc: 5/1,0 mode: PR/PR res: [0x240000403:0x2e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5faf1f expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.802518:0:2381:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ae8788d7/0x36f9bdc9e421c768 lrc: 4/0,0 mode: PR/PR res: [0x240000403:0x2e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5faf1f expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:3.0:1775764773.802538:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efde680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967006464:192.168.202.146@tcp:101:rm.0 00010000:00010000:3.0:1775764773.802563:0:2381:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005bcdc680/0x36f9bdc9e421c76f lrc: 6/2,0 mode: --/PR res: [0x280000403:0x2f:0x0].0x0 rrc: 2 type: EXT [2097152->8388607] (req 2097152->8388607) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.802573:0:2381:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005bcdc680/0x36f9bdc9e421c76f lrc: 6/2,0 mode: --/PR res: [0x280000403:0x2f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faf26 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.802580:0:2381:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005bcdc680/0x36f9bdc9e421c76f lrc: 6/2,0 mode: PR/PR res: [0x280000403:0x2f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5faf26 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.802588:0:2381:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005bcdc680/0x36f9bdc9e421c76f lrc: 6/2,0 mode: PR/PR res: [0x280000403:0x2f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faf26 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:3.0:1775764773.802608:0:2381:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 8388608, setting rss=8388608; kms=8388608, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005bcdc680/0x36f9bdc9e421c76f lrc: 5/2,0 mode: PR/PR res: [0x280000403:0x2f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5faf26 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:3.0:1775764773.802814:0:2381:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00010000:00010000:3.0:1775764773.802818:0:2381:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005bcdc680/0x36f9bdc9e421c76f lrc: 6/2,0 mode: PR/PR res: [0x280000403:0x2f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5faf26 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.802825:0:2381:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005bcdc680/0x36f9bdc9e421c76f lrc: 5/1,0 mode: PR/PR res: [0x280000403:0x2f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5faf26 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.802832:0:2381:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005bcdc680/0x36f9bdc9e421c76f lrc: 5/1,0 mode: PR/PR res: [0x280000403:0x2f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5faf26 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.802839:0:2381:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005bcdc680/0x36f9bdc9e421c76f lrc: 4/0,0 mode: PR/PR res: [0x280000403:0x2f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5faf26 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:3.0:1775764773.802852:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967006592:192.168.202.146@tcp:101:rm.0 00000008:00010000:1.0:1775764773.803002:0:2379:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 12582912, setting rss=12582912; kms=12582912, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005b4933ab/0x36f9bdc9e421c77d lrc: 5/2,0 mode: PR/PR res: [0x280000403:0x30:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5faf34 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:1.0:1775764773.803013:0:2379:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764773.803016:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005b4933ab/0x36f9bdc9e421c77d lrc: 6/2,0 mode: PR/PR res: [0x280000403:0x30:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5faf34 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764773.803023:0:2379:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005b4933ab/0x36f9bdc9e421c77d lrc: 5/1,0 mode: PR/PR res: [0x280000403:0x30:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5faf34 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764773.803031:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005b4933ab/0x36f9bdc9e421c77d lrc: 5/1,0 mode: PR/PR res: [0x280000403:0x30:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5faf34 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764773.803205:0:2379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005b4933ab/0x36f9bdc9e421c77d lrc: 4/0,0 mode: PR/PR res: [0x280000403:0x30:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5faf34 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:1.0:1775764773.803220:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967006848:192.168.202.146@tcp:101:rm.0 00010000:00010000:1.0:1775764773.803240:0:2379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c776 lrc: 6/2,0 mode: --/PR res: [0x240000403:0x30:0x0].0x0 rrc: 2 type: EXT [8388608->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764773.803251:0:2379:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c776 lrc: 6/2,0 mode: --/PR res: [0x240000403:0x30:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faf2d expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764773.803261:0:2379:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c776 lrc: 6/2,0 mode: PR/PR res: [0x240000403:0x30:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5faf2d expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764773.803270:0:2379:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c776 lrc: 6/2,0 mode: PR/PR res: [0x240000403:0x30:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faf2d expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1775764773.803281:0:2379:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 13631489, setting rss=13631489; kms=13631489, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c776 lrc: 5/2,0 mode: PR/PR res: [0x240000403:0x30:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5faf2d expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:1.0:1775764773.803289:0:2379:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:1.0:1775764773.803367:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c776 lrc: 6/2,0 mode: PR/PR res: [0x240000403:0x30:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5faf2d expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764773.803383:0:2379:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c776 lrc: 5/1,0 mode: PR/PR res: [0x240000403:0x30:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5faf2d expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764773.803391:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c776 lrc: 5/1,0 mode: PR/PR res: [0x240000403:0x30:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5faf2d expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764773.803398:0:2379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c776 lrc: 4/0,0 mode: PR/PR res: [0x240000403:0x30:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5faf2d expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:1.0:1775764773.803409:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967006720:192.168.202.146@tcp:101:rm.0 00000080:00200000:0.0:1775764773.803619:0:19896:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x262:0x0] updating i_size 26214401 i_blocks 53283 00000020:00010000:0.0:1775764773.803627:0:19896:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f997045afe8 (ffff8f994ef581b8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764773.803630:0:19896:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f997045afe8 (ffff8f994ef581b8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764773.803636:0:19896:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9966b03630 (ffff8f994ef58cf8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764773.803643:0:19896:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9966b03670 (ffff8f994ef587e8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764773.803646:0:19896:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9966b036b0 (ffff8f994ef58bd8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764773.803649:0:19896:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9966b036f0 (ffff8f994ef58b48/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764773.803652:0:19896:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9966b03730 (ffff8f994ef58128/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764773.803654:0:19896:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f997045afe8 (ffff8f994ef581b8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764773.803657:0:19896:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9966b03630 (ffff8f994ef581b8/1) at cl_lock_fini():72 00000020:00010000:0.0:1775764773.803662:0:19896:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9966b03670 (ffff8f994ef581b8/1) at cl_lock_fini():72 00000020:00010000:0.0:1775764773.803665:0:19896:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9966b036b0 (ffff8f994ef581b8/1) at cl_lock_fini():72 00000020:00010000:0.0:1775764773.803667:0:19896:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9966b036f0 (ffff8f994ef581b8/1) at cl_lock_fini():72 00000020:00010000:0.0:1775764773.803670:0:19896:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9966b03730 (ffff8f994ef581b8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764773.803769:0:19896:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x262:0x0] ignore/verify layout 0/1, layout version 5 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764773.803820:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764773.803831:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=client.txt, flags=0 00000080:00200000:0.0:1775764773.803836:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 3 00000080:00200000:0.0:1775764773.803840:0:19896:0:(namei.c:2395:ll_unlink()) VFS Op:name=client.txt, dir=[0x200000401:0x261:0x0](ffff8f9952e61988) 00010000:00010000:0.0:1775764773.803907:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c761 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x262:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5faf18 expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764773.804013:0:19896:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x262:0x0]: losing layout lock 00010000:00010000:0.0:1775764773.804204:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971226400/0x36f9bdc9e4211e88 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x262:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5d7635 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764773.804228:0:19896:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x262:0x0]: losing layout lock 00010000:00010000:0.0:1775764773.804269:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c761 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x262:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5faf18 expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764773.804283:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971226400/0x36f9bdc9e4211e88 lrc: 1/0,0 mode: --/CR res: [0x200000401:0x262:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5d7635 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764773.804288:0:19896:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1775764773.804290:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c761 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x262:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5faf18 expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764773.804450:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9971226400/0x36f9bdc9e4211e88 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x262:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5d7635 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000020:00100000:0.0:1775764773.804476:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1775764773.804487:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f0c8700 x1862022967007104/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764773.804510:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c8700 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967007104:192.168.202.146@tcp:36:rm.0 00000100:00100000:0.0:1775764773.804656:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16f180 going to sleep for 11 seconds 00000100:00100000:1.0:1775764773.814059:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c8700 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967007104:192.168.202.146@tcp:36:rm.0 00000080:00200000:1.0:1775764773.814579:0:19896:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x262:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764773.814601:0:19896:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:1.0:1775764773.814636:0:19896:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:1.0:1775764773.814666:0:19896:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x10 00000080:00200000:1.0:1775764773.814980:0:19896:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x262:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764773.847462:0:19896:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x262:0x0](ffff8f9952db1148) 00000008:00010000:0.0:1775764773.847538:0:19896:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 2097152 blocks 4103 [cma]time: 0 1775764044 0 by oinfo size 2097152 blocks 4103 [cma]time 0 1775764044 0 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ae8788d7/0x36f9bdc9e421c768 lrc: 1/0,0 mode: PR/PR res: [0x240000403:0x2e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5faf1f expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:0.0:1775764773.847566:0:19896:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 8388608 blocks 12295 [cma]time: 1775764046 1775764046 0 by oinfo size 8388608 blocks 12295 [cma]time 1775764046 1775764046 0 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005bcdc680/0x36f9bdc9e421c76f lrc: 1/0,0 mode: PR/PR res: [0x280000403:0x2f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5faf26 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:0.0:1775764773.847584:0:19896:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 8388608 blocks 16391 [cma]time: 1775764045 1775764046 0 by oinfo size 8388608 blocks 16391 [cma]time 1775764045 1775764046 1775764685 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000062300bdc/0x36f9bdc9e41c9bab lrc: 1/0,0 mode: PR/PR res: [0x240000403:0x2f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e4ebe4b expref: -99 pid: 19698 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00000008:00010000:0.0:1775764773.847617:0:19896:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 13631489 blocks 12295 [cma]time: 1775764046 1775764046 0 by oinfo size 13631489 blocks 12295 [cma]time 1775764046 1775764046 0 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c776 lrc: 1/0,0 mode: PR/PR res: [0x240000403:0x30:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5faf2d expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:0.0:1775764773.847632:0:19896:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 12582912 blocks 8199 [cma]time: 1775764046 1775764046 0 by oinfo size 12582912 blocks 8199 [cma]time 1775764046 1775764046 0 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005b4933ab/0x36f9bdc9e421c77d lrc: 1/0,0 mode: PR/PR res: [0x280000403:0x30:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5faf34 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764773.847729:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:0.0:1775764773.847869:0:19896:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=dbench, dir=[0x200000401:0x261:0x0](ffff8f9952e61988), flags=0 00000080:00200000:0.0:1775764773.847874:0:19896:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=dbench, dir=[0x200000401:0x261:0x0](ffff8f9952e61988), intent=getattr 00000002:00010000:0.0:1775764773.848203:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: dbench,[0x0:0x0:0x0]) in obj [0x200000401:0x261:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764773.848266:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c784 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.848276:0:19896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c784 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764773.848285:0:19896:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c784 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764773.848465:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c8000 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967007232:192.168.202.146@tcp:101:rm.0 00000100:00100000:0.0:1775764773.848650:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f16fa80 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.856575:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c8000 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967007232:192.168.202.146@tcp:101:rm.0 00010000:00010000:2.0:1775764773.856623:0:19896:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c784 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faf6c expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.856653:0:19896:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c784 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x263:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faf6c expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.856663:0:19896:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c784 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x263:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faf6c expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.856675:0:19896:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c784 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x263:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5faf6c expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.856687:0:19896:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c784 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x263:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faf6c expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.856700:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c784 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x263:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5faf6c expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.856708:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c784 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x263:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5faf6c expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.856714:0:19896:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c784 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x263:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5faf6c expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764773.856723:0:19896:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994f0c8000 x1862022967007232/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/928 e 0 to 0 dl 1775764789 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764773.856734:0:19896:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c784 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x263:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faf6c expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764773.856782:0:19896:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c784 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x263:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faf6c expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.856793:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/611 (0 0) 00000080:00200000:2.0:1775764773.856798:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x261:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764773.856909:0:19896:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x263:0x0](ffff8f9952e62a08): rc = 0 00000080:00200000:2.0:1775764773.856939:0:19896:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x263:0x0]: layout version change: 3 -> 3 00000080:00010000:2.0:1775764773.856949:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x263:0x0](ffff8f9952e62a08) for lock 0x36f9bdc9e421c784 00000080:00010000:2.0:1775764773.856960:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x263:0x0](ffff8f9952e62a08) 00000080:00010000:2.0:1775764773.856963:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c784 from it ffffafbac1ec7b20 00010000:00010000:2.0:1775764773.856967:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c784 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x263:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5faf6c expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.856982:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c784 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x263:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5faf6c expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200008:2.0:1775764773.857028:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file dbench:[0x200000401:0x263:0x0](ffff8f9952e62a08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764773.857045:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x263:0x0](ffff8f9952e62a08),name=dbench 00000002:00010000:2.0:1775764773.857067:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x263:0x0]) in obj [0x200000401:0x261:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764773.857076:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c784 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x263:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5faf6c expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764773.857086:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c784 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x263:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5faf6c expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764773.857093:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x261:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764773.857097:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x263:0x0](ffff8f9952e62a08) 00000080:00010000:2.0:1775764773.857100:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x263:0x0](ffff8f9952e62a08) for lock 0x36f9bdc9e421c784 00000080:00010000:2.0:1775764773.857103:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c784 from it ffffafbac1ec7cb8 00010000:00010000:2.0:1775764773.857106:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c784 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x263:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5faf6c expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764773.857113:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c784 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x263:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5faf6c expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764773.857468:0:19896:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x263:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1775764773.857479:0:19896:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x263:0x0] 00000008:00010000:2.0:1775764773.857505:0:19896:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764773.857508:0:19896:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994166f3b0, osc lock ffff8f994f1a3b50, flags 0x400001002 00010000:00010000:2.0:1775764773.857525:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f9afe1c6/0x36f9bdc9e4211ece lrc: 2/1,0 mode: PR/PR res: [0x240000403:0x31:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5d76f9 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764773.857536:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f9afe1c6/0x36f9bdc9e4211ece lrc: 2/1,0 mode: PR/PR res: [0x240000403:0x31:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5d76f9 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:2.0:1775764773.857547:0:19896:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1775764773.857550:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f9afe1c6/0x36f9bdc9e4211ece lrc: 5/1,0 mode: PR/PR res: [0x240000403:0x31:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5d76f9 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764773.857558:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f9afe1c6/0x36f9bdc9e4211ece lrc: 4/0,0 mode: PR/PR res: [0x240000403:0x31:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5d76f9 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:2.0:1775764773.857568:0:19896:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1775764773.857578:0:19896:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x263:0x0] updating i_size 51544 i_blocks 131 00000020:00010000:2.0:1775764773.857582:0:19896:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f997045a6c8 (ffff8f994ef58a28/0) at cl_lock_release():245 00000020:00010000:2.0:1775764773.857585:0:19896:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f997045a6c8 (ffff8f994ef58a28/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764773.857589:0:19896:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994166f3b0 (ffff8f994ef585a8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764773.857602:0:19896:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f997045a6c8 (ffff8f994ef58a28/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764773.857605:0:19896:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994166f3b0 (ffff8f994ef58a28/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764773.857625:0:19896:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x263:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764773.857663:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:2.0:1775764773.857673:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=dbench, flags=0 00000080:00200000:2.0:1775764773.857678:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 3 00000080:00200000:2.0:1775764773.857682:0:19896:0:(namei.c:2395:ll_unlink()) VFS Op:name=dbench, dir=[0x200000401:0x261:0x0](ffff8f9952e61988) 00010000:00010000:2.0:1775764773.857716:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c784 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x263:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5faf6c expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764773.857768:0:19896:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x263:0x0]: losing layout lock 00010000:00010000:2.0:1775764773.857790:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925800/0x36f9bdc9e4211ec7 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x263:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5d76f2 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764773.857806:0:19896:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x263:0x0]: losing layout lock 00010000:00010000:2.0:1775764773.857840:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c784 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x263:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5faf6c expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764773.857848:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925800/0x36f9bdc9e4211ec7 lrc: 1/0,0 mode: --/CR res: [0x200000401:0x263:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5d76f2 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764773.857854:0:19896:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:2.0:1775764773.857857:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c784 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x263:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5faf6c expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764773.857869:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970925800/0x36f9bdc9e4211ec7 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x263:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5d76f2 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000020:00100000:2.0:1775764773.857888:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1775764773.857896:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994efddf80 x1862022967007488/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 488/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764773.857920:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efddf80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967007488:192.168.202.146@tcp:36:rm.0 00000100:00100000:2.0:1775764773.858508:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102c00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.867570:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efddf80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967007488:192.168.202.146@tcp:36:rm.0 00000080:00200000:2.0:1775764773.867679:0:19896:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x263:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764773.867690:0:19896:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:2.0:1775764773.867712:0:19896:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:2.0:1775764773.867714:0:19896:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:2.0:1775764773.867768:0:19896:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x263:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764773.867887:0:19896:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x263:0x0](ffff8f9952e62a08) 00000008:00010000:2.0:1775764773.868338:0:19896:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 51544 blocks 131 [cma]time: 1775764047 1775764047 0 by oinfo size 51544 blocks 131 [cma]time 1775764047 1775764047 0 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f9afe1c6/0x36f9bdc9e4211ece lrc: 1/0,0 mode: PR/PR res: [0x240000403:0x31:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5d76f9 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764773.868441:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:2.0:1775764773.868458:0:19896:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=lib64, dir=[0x200000401:0x261:0x0](ffff8f9952e61988), flags=16 00000080:00200000:2.0:1775764773.868464:0:19896:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=lib64, dir=[0x200000401:0x261:0x0](ffff8f9952e61988), intent=0 00000002:00010000:2.0:1775764773.868487:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: lib64,[0x0:0x0:0x0]) in obj [0x200000401:0x261:0x0], intent: lookup flags 0 00010000:00010000:2.0:1775764773.868557:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c78b lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.868567:0:19896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c78b lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.868577:0:19896:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c78b lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764773.868607:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdf800 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967007616:192.168.202.146@tcp:101:rm.0 00000100:00100000:2.0:1775764773.868706:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.877093:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdf800 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967007616:192.168.202.146@tcp:101:rm.0 00010000:00010000:2.0:1775764773.877118:0:19896:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c78b lrc: 4/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fafa4 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.877157:0:19896:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c78b lrc: 4/1,0 mode: --/PR res: [0x200000401:0x264:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fafa4 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.877170:0:19896:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c78b lrc: 4/1,0 mode: --/PR res: [0x200000401:0x264:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fafa4 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.877181:0:19896:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c78b lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x264:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fafa4 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.877188:0:19896:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c78b lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x264:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fafa4 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.877198:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c78b lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x264:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fafa4 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.877203:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c78b lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x264:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fafa4 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.877207:0:19896:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c78b lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x264:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fafa4 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764773.877215:0:19896:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=b, status=0 req@ffff8f994efdf800 x1862022967007616/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/584 e 0 to 0 dl 1775764789 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764773.877226:0:19896:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c78b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x264:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fafa4 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.877233:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/612 (0 0) 00000080:00200000:2.0:1775764773.877239:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x261:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764773.877258:0:19896:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x264:0x0], updating i_size 7168 00000080:00200000:2.0:1775764773.877261:0:19896:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x264:0x0](ffff8f9952da1988): rc = 0 00000080:00010000:2.0:1775764773.877264:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x264:0x0](ffff8f9952da1988) for lock 0x36f9bdc9e421c78b 00000080:00010000:2.0:1775764773.877274:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x264:0x0](ffff8f9952da1988) 00000080:00010000:2.0:1775764773.877277:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c78b from it ffffafbac1ec7a28 00010000:00010000:2.0:1775764773.877279:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c78b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x264:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fafa4 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.877285:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c78b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x264:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fafa4 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764773.877326:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x264:0x0](ffff8f9952da1988), inode mode 41ed mask 1 00000080:00200000:2.0:1775764773.877344:0:19896:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=libpopt.so.0, dir=[0x200000401:0x264:0x0](ffff8f9952da1988), flags=0 00000080:00200000:2.0:1775764773.877347:0:19896:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=libpopt.so.0, dir=[0x200000401:0x264:0x0](ffff8f9952da1988), intent=getattr 00000002:00010000:2.0:1775764773.877376:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: libpopt.so.0,[0x0:0x0:0x0]) in obj [0x200000401:0x264:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764773.877410:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c792 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x264:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.877415:0:19896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c792 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x264:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.877421:0:19896:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c792 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x264:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764773.877435:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdf800 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967007744:192.168.202.146@tcp:101:rm.0 00000100:00100000:2.0:1775764773.877520:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.888061:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdf800 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967007744:192.168.202.146@tcp:101:rm.0 00010000:00010000:2.0:1775764773.888088:0:19896:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c792 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x264:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fafc0 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.888110:0:19896:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c792 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x265:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fafc0 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.888118:0:19896:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c792 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x265:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fafc0 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.888139:0:19896:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c792 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x265:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fafc0 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.888148:0:19896:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c792 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x265:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fafc0 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.888159:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c792 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x265:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fafc0 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.888167:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c792 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x265:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fafc0 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.888173:0:19896:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c792 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x265:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fafc0 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764773.888181:0:19896:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994efdf800 x1862022967007744/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/928 e 0 to 0 dl 1775764789 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764773.888190:0:19896:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c792 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x265:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fafc0 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764773.888208:0:19896:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c792 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x265:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fafc0 expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.888217:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/613 (0 0) 00000080:00200000:2.0:1775764773.888223:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x264:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764773.888315:0:19896:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x265:0x0](ffff8f9952da1148): rc = 0 00000080:00200000:2.0:1775764773.888335:0:19896:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x265:0x0]: layout version change: 3 -> 3 00000080:00010000:2.0:1775764773.888345:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x265:0x0](ffff8f9952da1148) for lock 0x36f9bdc9e421c792 00000080:00010000:2.0:1775764773.888367:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x265:0x0](ffff8f9952da1148) 00000080:00010000:2.0:1775764773.888372:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c792 from it ffffafbac1ec7b20 00010000:00010000:2.0:1775764773.888375:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c792 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x265:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fafc0 expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.888384:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c792 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x265:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fafc0 expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200008:2.0:1775764773.888434:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file libpopt.so.0:[0x200000401:0x265:0x0](ffff8f9952da1148), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764773.888446:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x265:0x0](ffff8f9952da1148),name=libpopt.so.0 00000002:00010000:2.0:1775764773.888467:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x265:0x0]) in obj [0x200000401:0x264:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764773.888475:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c792 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x265:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fafc0 expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764773.888485:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c792 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x265:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fafc0 expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764773.888492:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x264:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764773.888497:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x265:0x0](ffff8f9952da1148) 00000080:00010000:2.0:1775764773.888500:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x265:0x0](ffff8f9952da1148) for lock 0x36f9bdc9e421c792 00000080:00010000:2.0:1775764773.888503:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c792 from it ffffafbac1ec7cb8 00010000:00010000:2.0:1775764773.888506:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c792 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x265:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fafc0 expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764773.888513:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c792 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x265:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fafc0 expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764773.888538:0:19896:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x265:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1775764773.888547:0:19896:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x265:0x0] 00000008:00010000:2.0:1775764773.888572:0:19896:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764773.888576:0:19896:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994166f3b0, osc lock ffff8f994f1a3b50, flags 0x400001002 00010000:00010000:2.0:1775764773.888602:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000aa9eb34d/0x36f9bdc9e421c179 lrc: 2/1,0 mode: PR/PR res: [0x280000403:0x31:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5f889f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764773.888613:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000aa9eb34d/0x36f9bdc9e421c179 lrc: 2/1,0 mode: PR/PR res: [0x280000403:0x31:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5f889f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:2.0:1775764773.888626:0:19896:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1775764773.888629:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000aa9eb34d/0x36f9bdc9e421c179 lrc: 5/1,0 mode: PR/PR res: [0x280000403:0x31:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5f889f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764773.888636:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000aa9eb34d/0x36f9bdc9e421c179 lrc: 4/0,0 mode: PR/PR res: [0x280000403:0x31:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5f889f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:2.0:1775764773.888646:0:19896:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1775764773.888655:0:19896:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x265:0x0] updating i_size 54680 i_blocks 131 00000020:00010000:2.0:1775764773.888661:0:19896:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f997045a6c8 (ffff8f994ef58a28/0) at cl_lock_release():245 00000020:00010000:2.0:1775764773.888664:0:19896:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f997045a6c8 (ffff8f994ef58a28/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764773.888666:0:19896:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994166f3b0 (ffff8f994ef585a8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764773.888670:0:19896:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f997045a6c8 (ffff8f994ef58a28/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764773.888673:0:19896:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994166f3b0 (ffff8f994ef58a28/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764773.888691:0:19896:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x265:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764773.888736:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:2.0:1775764773.888784:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=lib64, flags=16 00000080:00200000:2.0:1775764773.888790:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x264:0x0](ffff8f9952da1988), inode mode 41ed mask 1 00000080:00200000:2.0:1775764773.888797:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=libpopt.so.0, flags=0 00000080:00200000:2.0:1775764773.888801:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x264:0x0](ffff8f9952da1988), inode mode 41ed mask 3 00000080:00200000:2.0:1775764773.888805:0:19896:0:(namei.c:2395:ll_unlink()) VFS Op:name=libpopt.so.0, dir=[0x200000401:0x264:0x0](ffff8f9952da1988) 00010000:00010000:2.0:1775764773.888843:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c78b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x264:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fafa4 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764773.888872:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c792 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x265:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5fafc0 expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764773.888887:0:19896:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x265:0x0]: losing layout lock 00010000:00010000:2.0:1775764773.888900:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d8e00/0x36f9bdc9e421c172 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x265:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f8898 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1775764773.888914:0:19896:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x265:0x0]: losing layout lock 00010000:00010000:2.0:1775764773.888945:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c792 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x265:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fafc0 expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764773.888953:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d8e00/0x36f9bdc9e421c172 lrc: 1/0,0 mode: --/CR res: [0x200000401:0x265:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f8898 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764773.888965:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c78b lrc: 1/0,0 mode: --/PR res: [0x200000401:0x264:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fafa4 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764773.888971:0:19896:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:2.0:1775764773.888974:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c792 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x265:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fafc0 expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764773.888987:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d8e00/0x36f9bdc9e421c172 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x265:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f8898 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764773.889002:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c78b lrc: 0/0,0 mode: --/PR res: [0x200000401:0x264:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fafa4 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000020:00100000:2.0:1775764773.889019:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1775764773.889027:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994efdf800 x1862022967008000/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 504/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764773.889049:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdf800 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967008000:192.168.202.146@tcp:36:rm.0 00000100:00100000:2.0:1775764773.889133:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.898069:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdf800 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967008000:192.168.202.146@tcp:36:rm.0 00000080:00200000:2.0:1775764773.898174:0:19896:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x265:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764773.898189:0:19896:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:2.0:1775764773.898208:0:19896:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:2.0:1775764773.898210:0:19896:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:2.0:1775764773.898244:0:19896:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x265:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764773.898359:0:19896:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x265:0x0](ffff8f9952da1148) 00000008:00010000:2.0:1775764773.898402:0:19896:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 54680 blocks 131 [cma]time: 1775764047 1621392726 0 by oinfo size 54680 blocks 131 [cma]time 1775764047 1621392726 0 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000aa9eb34d/0x36f9bdc9e421c179 lrc: 1/0,0 mode: PR/PR res: [0x280000403:0x31:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5f889f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764773.898517:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:2.0:1775764773.898533:0:19896:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=lib64, dir=[0x200000401:0x261:0x0](ffff8f9952e61988), flags=16 00000080:00200000:2.0:1775764773.898538:0:19896:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=lib64, dir=[0x200000401:0x261:0x0](ffff8f9952e61988), intent=0 00000002:00010000:2.0:1775764773.898558:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: lib64,[0x0:0x0:0x0]) in obj [0x200000401:0x261:0x0], intent: lookup flags 0 00010000:00010000:2.0:1775764773.898633:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c799 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.898642:0:19896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c799 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.898652:0:19896:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c799 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764773.898677:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdf480 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967008128:192.168.202.146@tcp:101:rm.0 00000100:00100000:2.0:1775764773.898821:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.912512:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdf480 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967008128:192.168.202.146@tcp:101:rm.0 00010000:00010000:2.0:1775764773.912547:0:19896:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c799 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faff8 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.912584:0:19896:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c799 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x264:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faff8 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.912601:0:19896:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c799 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x264:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faff8 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.912621:0:19896:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c799 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x264:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5faff8 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.912635:0:19896:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c799 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x264:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faff8 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.912649:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c799 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x264:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5faff8 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.912657:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c799 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x264:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5faff8 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.912665:0:19896:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c799 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x264:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5faff8 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764773.912678:0:19896:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=b, status=0 req@ffff8f994efdf480 x1862022967008128/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/584 e 0 to 0 dl 1775764789 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764773.912694:0:19896:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c799 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x264:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faff8 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.912706:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/612 (0 0) 00000080:00200000:2.0:1775764773.912714:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 16 on i1 [0x200000401:0x261:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764773.912770:0:19896:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x264:0x0], updating i_size 7168 00000080:00200000:2.0:1775764773.912775:0:19896:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x264:0x0](ffff8f9952da1988): rc = 0 00000080:00010000:2.0:1775764773.912781:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x264:0x0](ffff8f9952da1988) for lock 0x36f9bdc9e421c799 00000080:00010000:2.0:1775764773.912797:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x264:0x0](ffff8f9952da1988) 00000080:00010000:2.0:1775764773.912801:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c799 from it ffffafbac1ec7a28 00010000:00010000:2.0:1775764773.912804:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c799 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x264:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5faff8 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.912818:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c799 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x264:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5faff8 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764773.912867:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x264:0x0](ffff8f9952da1988), inode mode 41ed mask 1 00000080:00200000:2.0:1775764773.912897:0:19896:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=libc.so.6, dir=[0x200000401:0x264:0x0](ffff8f9952da1988), flags=0 00000080:00200000:2.0:1775764773.912903:0:19896:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=libc.so.6, dir=[0x200000401:0x264:0x0](ffff8f9952da1988), intent=getattr 00000002:00010000:2.0:1775764773.912929:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: libc.so.6,[0x0:0x0:0x0]) in obj [0x200000401:0x264:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764773.912979:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c7a0 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x264:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.912986:0:19896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c7a0 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x264:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.912996:0:19896:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c7a0 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x264:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764773.913022:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdf480 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967008256:192.168.202.146@tcp:101:rm.0 00000100:00100000:2.0:1775764773.913108:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.916808:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdf480 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967008256:192.168.202.146@tcp:101:rm.0 00010000:00010000:2.0:1775764773.916835:0:19896:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c7a0 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x264:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb014 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.916851:0:19896:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c7a0 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x266:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb014 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.916858:0:19896:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c7a0 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x266:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb014 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.916873:0:19896:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c7a0 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x266:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb014 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.916884:0:19896:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c7a0 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x266:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb014 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.916895:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c7a0 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x266:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb014 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.916903:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c7a0 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x266:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb014 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.916909:0:19896:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c7a0 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x266:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb014 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764773.916926:0:19896:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994efdf480 x1862022967008256/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/952 e 0 to 0 dl 1775764789 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764773.916937:0:19896:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 368 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c7a0 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x266:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb014 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764773.916950:0:19896:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c7a0 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x266:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb014 expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.916961:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/614 (0 0) 00000080:00200000:2.0:1775764773.916967:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x264:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764773.917056:0:19896:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x266:0x0](ffff8f9952e642c8): rc = 0 00000080:00200000:2.0:1775764773.917079:0:19896:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x266:0x0]: layout version change: 4 -> 4 00000080:00010000:2.0:1775764773.917090:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x266:0x0](ffff8f9952e642c8) for lock 0x36f9bdc9e421c7a0 00000080:00010000:2.0:1775764773.917104:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x266:0x0](ffff8f9952e642c8) 00000080:00010000:2.0:1775764773.917107:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c7a0 from it ffffafbac1ec7b20 00010000:00010000:2.0:1775764773.917111:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c7a0 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x266:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fb014 expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.917131:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c7a0 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x266:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fb014 expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200008:2.0:1775764773.917181:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file libc.so.6:[0x200000401:0x266:0x0](ffff8f9952e642c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764773.917192:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x266:0x0](ffff8f9952e642c8),name=libc.so.6 00000002:00010000:2.0:1775764773.917212:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x266:0x0]) in obj [0x200000401:0x264:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764773.917221:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c7a0 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x266:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fb014 expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764773.917231:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c7a0 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x266:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fb014 expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764773.917237:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x264:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764773.917242:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x266:0x0](ffff8f9952e642c8) 00000080:00010000:2.0:1775764773.917245:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x266:0x0](ffff8f9952e642c8) for lock 0x36f9bdc9e421c7a0 00000080:00010000:2.0:1775764773.917248:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c7a0 from it ffffafbac1ec7cb8 00010000:00010000:2.0:1775764773.917250:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c7a0 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x266:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fb014 expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764773.917256:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c7a0 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x266:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fb014 expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764773.917286:0:19896:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x266:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1775764773.917296:0:19896:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x266:0x0] 00000008:00010000:2.0:1775764773.917323:0:19896:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764773.917327:0:19896:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f9943053930, osc lock ffff8f994f1a3b50, flags 0x400001002 00000008:00010000:2.0:1775764773.917344:0:19896:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764773.917346:0:19896:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f9943053970, osc lock ffff8f994f1a38f0, flags 0x400001002 00000008:00010000:2.0:1775764773.917372:0:19896:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764773.917374:0:19896:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99430539b0, osc lock ffff8f994f1a2ab0, flags 0x400001002 00010000:00010000:2.0:1775764773.917390:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4f14cf5/0x36f9bdc9e4211ec0 lrc: 2/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 1232896->1236991) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 0 00010000:00010000:2.0:1775764773.917402:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4f14cf5/0x36f9bdc9e4211ec0 lrc: 2/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1232896->1236991) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 0 00000020:00200000:2.0:1775764773.917414:0:19896:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1775764773.917418:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4f14cf5/0x36f9bdc9e4211ec0 lrc: 5/1,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1232896->1236991) gid 0 flags: 0x810020000020000 nid: local remote: 0x2aac6a50e5d76eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 0 00010000:00010000:2.0:1775764773.917426:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4f14cf5/0x36f9bdc9e4211ec0 lrc: 4/0,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1232896->1236991) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 0 00010000:00010000:2.0:1775764773.917470:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000015339cf1/0x36f9bdc9e421c7a7 lrc: 3/1,0 mode: --/PR res: [0x280000403:0x32:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.917478:0:19896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000015339cf1/0x36f9bdc9e421c7a7 lrc: 3/1,0 mode: --/PR res: [0x280000403:0x32:0x0].0x0 rrc: 2 type: EXT [2097152->8388607] (req 2097152->8388607) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.917536:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c8654ade/0x36f9bdc9e421c7ae lrc: 3/1,0 mode: --/PR res: [0x240000403:0x33:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.917544:0:19896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c8654ade/0x36f9bdc9e421c7ae lrc: 3/1,0 mode: --/PR res: [0x240000403:0x33:0x0].0x0 rrc: 2 type: EXT [0->8388607] (req 0->8388607) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:2.0:1775764773.917554:0:19896:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000100:00100000:3.0:1775764773.918352:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967008512:192.168.202.146@tcp:101:rm.0 00000100:00100000:3.0:1775764773.918455:0:2382:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1775764773.918462:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967008384:192.168.202.146@tcp:101:rm.0 00010000:00010000:3.0:1775764773.930921:0:2382:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000015339cf1/0x36f9bdc9e421c7a7 lrc: 6/2,0 mode: --/PR res: [0x280000403:0x32:0x0].0x0 rrc: 2 type: EXT [2097152->8388607] (req 2097152->8388607) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.930954:0:2382:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000015339cf1/0x36f9bdc9e421c7a7 lrc: 6/2,0 mode: --/PR res: [0x280000403:0x32:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb022 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.930969:0:2382:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000015339cf1/0x36f9bdc9e421c7a7 lrc: 6/2,0 mode: PR/PR res: [0x280000403:0x32:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb022 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.930982:0:2382:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000015339cf1/0x36f9bdc9e421c7a7 lrc: 6/2,0 mode: PR/PR res: [0x280000403:0x32:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb022 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:3.0:1775764773.930997:0:2382:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000015339cf1/0x36f9bdc9e421c7a7 lrc: 5/2,0 mode: PR/PR res: [0x280000403:0x32:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fb022 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:3.0:1775764773.931009:0:2382:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1775764773.931013:0:2382:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000015339cf1/0x36f9bdc9e421c7a7 lrc: 6/2,0 mode: PR/PR res: [0x280000403:0x32:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fb022 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.931021:0:2382:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000015339cf1/0x36f9bdc9e421c7a7 lrc: 5/1,0 mode: PR/PR res: [0x280000403:0x32:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fb022 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.931029:0:2382:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000015339cf1/0x36f9bdc9e421c7a7 lrc: 5/1,0 mode: PR/PR res: [0x280000403:0x32:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fb022 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.931039:0:2382:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000015339cf1/0x36f9bdc9e421c7a7 lrc: 4/0,0 mode: PR/PR res: [0x280000403:0x32:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fb022 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:3.0:1775764773.931065:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967008384:192.168.202.146@tcp:101:rm.0 00010000:00010000:3.0:1775764773.931083:0:2382:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c8654ade/0x36f9bdc9e421c7ae lrc: 6/2,0 mode: --/PR res: [0x240000403:0x33:0x0].0x0 rrc: 2 type: EXT [0->8388607] (req 0->8388607) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.931093:0:2382:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c8654ade/0x36f9bdc9e421c7ae lrc: 6/2,0 mode: --/PR res: [0x240000403:0x33:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8388607) gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb01b expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.931101:0:2382:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c8654ade/0x36f9bdc9e421c7ae lrc: 6/2,0 mode: PR/PR res: [0x240000403:0x33:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8388607) gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb01b expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.931110:0:2382:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c8654ade/0x36f9bdc9e421c7ae lrc: 6/2,0 mode: PR/PR res: [0x240000403:0x33:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8388607) gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb01b expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:3.0:1775764773.931119:0:2382:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 71696, setting rss=71696; kms=71696, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c8654ade/0x36f9bdc9e421c7ae lrc: 5/2,0 mode: PR/PR res: [0x240000403:0x33:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8388607) gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fb01b expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:3.0:1775764773.931139:0:2382:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:3.0:1775764773.931156:0:2382:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c8654ade/0x36f9bdc9e421c7ae lrc: 6/2,0 mode: PR/PR res: [0x240000403:0x33:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8388607) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fb01b expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.931164:0:2382:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c8654ade/0x36f9bdc9e421c7ae lrc: 5/1,0 mode: PR/PR res: [0x240000403:0x33:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8388607) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fb01b expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.931171:0:2382:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c8654ade/0x36f9bdc9e421c7ae lrc: 5/1,0 mode: PR/PR res: [0x240000403:0x33:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8388607) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fb01b expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.931180:0:2382:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c8654ade/0x36f9bdc9e421c7ae lrc: 4/0,0 mode: PR/PR res: [0x240000403:0x33:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fb01b expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:3.0:1775764773.931190:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967008512:192.168.202.146@tcp:101:rm.0 00000080:00200000:2.0:1775764773.931294:0:19896:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x266:0x0] updating i_size 2168848 i_blocks 4363 00000020:00010000:2.0:1775764773.931301:0:19896:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f997045a6c8 (ffff8f994ef58a28/0) at cl_lock_release():245 00000020:00010000:2.0:1775764773.931304:0:19896:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f997045a6c8 (ffff8f994ef58a28/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764773.931309:0:19896:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9943053930 (ffff8f994ef585a8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764773.931313:0:19896:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9943053970 (ffff8f994ef582d8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764773.931316:0:19896:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99430539b0 (ffff8f994ef58908/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764773.931318:0:19896:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f997045a6c8 (ffff8f994ef58a28/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764773.931321:0:19896:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9943053930 (ffff8f994ef58a28/1) at cl_lock_fini():72 00000020:00010000:2.0:1775764773.931325:0:19896:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9943053970 (ffff8f994ef58a28/1) at cl_lock_fini():72 00000020:00010000:2.0:1775764773.931327:0:19896:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99430539b0 (ffff8f994ef58a28/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764773.931397:0:19896:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x266:0x0] ignore/verify layout 0/1, layout version 4 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764773.931456:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:2.0:1775764773.931467:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=lib64, flags=16 00000080:00200000:2.0:1775764773.931475:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x264:0x0](ffff8f9952da1988), inode mode 41ed mask 1 00000080:00200000:2.0:1775764773.931482:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=libc.so.6, flags=0 00000080:00200000:2.0:1775764773.931486:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x264:0x0](ffff8f9952da1988), inode mode 41ed mask 3 00000080:00200000:2.0:1775764773.931492:0:19896:0:(namei.c:2395:ll_unlink()) VFS Op:name=libc.so.6, dir=[0x200000401:0x264:0x0](ffff8f9952da1988) 00010000:00010000:2.0:1775764773.931575:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c7a0 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x266:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5fb014 expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764773.931628:0:19896:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x266:0x0]: losing layout lock 00010000:00010000:2.0:1775764773.931652:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970924e00/0x36f9bdc9e4211eb2 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x266:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5d76dd expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764773.931668:0:19896:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x266:0x0]: losing layout lock 00010000:00010000:2.0:1775764773.931706:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c7a0 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x266:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fb014 expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764773.931713:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970924e00/0x36f9bdc9e4211eb2 lrc: 1/0,0 mode: --/CR res: [0x200000401:0x266:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5d76dd expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764773.931719:0:19896:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:2.0:1775764773.931722:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c7a0 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x266:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fb014 expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764773.931782:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970924e00/0x36f9bdc9e4211eb2 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x266:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5d76dd expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000020:00100000:2.0:1775764773.931803:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1775764773.931810:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994efde680 x1862022967008768/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764773.931834:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efde680 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967008768:192.168.202.146@tcp:36:rm.0 00000100:00100000:2.0:1775764773.931906:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.936267:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efde680 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967008768:192.168.202.146@tcp:36:rm.0 00000080:00200000:2.0:1775764773.936364:0:19896:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x266:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764773.936378:0:19896:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:2.0:1775764773.936403:0:19896:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:2.0:1775764773.936433:0:19896:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:2.0:1775764773.936489:0:19896:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x266:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764773.937040:0:19896:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x266:0x0](ffff8f9952e642c8) 00000008:00010000:2.0:1775764773.937093:0:19896:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 2097152 blocks 4103 [cma]time: 1775764048 1749497575 1775764049 by oinfo size 2097152 blocks 4103 [cma]time 1775764048 1749497575 1775764049 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4f14cf5/0x36f9bdc9e4211ec0 lrc: 1/0,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 1232896->1236991) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 0 00000008:00010000:2.0:1775764773.937111:0:19896:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 2097152 blocks 4103 [cma]time: 1775764048 1749497575 1775764049 by oinfo size 2097152 blocks 4103 [cma]time 1775764048 1749497575 1775764049 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 1/0,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 4 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x800020000020000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 0 00000008:00010000:2.0:1775764773.937150:0:19896:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 1775764048 1749497575 0 by oinfo size 0 blocks 1 [cma]time 1775764048 1749497575 0 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000015339cf1/0x36f9bdc9e421c7a7 lrc: 1/0,0 mode: PR/PR res: [0x280000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fb022 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:2.0:1775764773.937168:0:19896:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 71696 blocks 259 [cma]time: 1775764048 1749497575 0 by oinfo size 71696 blocks 259 [cma]time 1775764048 1749497575 0 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c8654ade/0x36f9bdc9e421c7ae lrc: 1/0,0 mode: PR/PR res: [0x240000403:0x33:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fb01b expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764773.937266:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:2.0:1775764773.937275:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=lib64, flags=16 00000080:00200000:2.0:1775764773.937282:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x264:0x0](ffff8f9952da1988), inode mode 41ed mask 1 00000080:00200000:2.0:1775764773.937293:0:19896:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=ld-linux-x86-64.so.2, dir=[0x200000401:0x264:0x0](ffff8f9952da1988), flags=0 00000080:00200000:2.0:1775764773.937297:0:19896:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=ld-linux-x86-64.so.2, dir=[0x200000401:0x264:0x0](ffff8f9952da1988), intent=getattr 00000002:00010000:2.0:1775764773.937317:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ld-linux-x86-64.so.2,[0x0:0x0:0x0]) in obj [0x200000401:0x264:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764773.937371:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c7b5 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x264:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.937386:0:19896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c7b5 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x264:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.937396:0:19896:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c7b5 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x264:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19896 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764773.937419:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdc700 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967008896:192.168.202.146@tcp:101:rm.0 00000100:00100000:2.0:1775764773.937513:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.952846:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdc700 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967008896:192.168.202.146@tcp:101:rm.0 00010000:00010000:2.0:1775764773.952878:0:19896:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c7b5 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x264:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb05a expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.952897:0:19896:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c7b5 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x267:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb05a expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.952905:0:19896:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c7b5 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x267:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb05a expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.952922:0:19896:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c7b5 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x267:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb05a expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.952934:0:19896:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c7b5 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x267:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb05a expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.952949:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c7b5 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x267:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb05a expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.952963:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c7b5 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x267:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb05a expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.952969:0:19896:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c7b5 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x267:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb05a expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764773.952981:0:19896:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994efdc700 x1862022967008896/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 592/928 e 0 to 0 dl 1775764789 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764773.952993:0:19896:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c7b5 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x267:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb05a expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764773.953010:0:19896:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c7b5 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x267:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb05a expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.953020:0:19896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/615 (0 0) 00000080:00200000:2.0:1775764773.953027:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x264:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764773.953128:0:19896:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x267:0x0](ffff8f9952e63248): rc = 0 00000080:00200000:2.0:1775764773.953148:0:19896:0:(file.c:6766:ll_layout_conf()) [0x200000401:0x267:0x0]: layout version change: 3 -> 3 00000080:00010000:2.0:1775764773.953157:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x267:0x0](ffff8f9952e63248) for lock 0x36f9bdc9e421c7b5 00000080:00010000:2.0:1775764773.953168:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x267:0x0](ffff8f9952e63248) 00000080:00010000:2.0:1775764773.953172:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c7b5 from it ffffafbac1ec7b20 00010000:00010000:2.0:1775764773.953176:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c7b5 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x267:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fb05a expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.953185:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c7b5 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x267:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fb05a expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200008:2.0:1775764773.953238:0:19896:0:(file.c:6226:ll_getattr_dentry()) START file ld-linux-x86-64.so.2:[0x200000401:0x267:0x0](ffff8f9952e63248), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764773.953249:0:19896:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x267:0x0](ffff8f9952e63248),name=ld-linux-x86-64.so.2 00000002:00010000:2.0:1775764773.953268:0:19896:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x267:0x0]) in obj [0x200000401:0x264:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764773.953277:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c7b5 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x267:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fb05a expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764773.953288:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c7b5 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x267:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fb05a expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764773.953295:0:19896:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x264:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764773.953299:0:19896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x267:0x0](ffff8f9952e63248) 00000080:00010000:2.0:1775764773.953303:0:19896:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x267:0x0](ffff8f9952e63248) for lock 0x36f9bdc9e421c7b5 00000080:00010000:2.0:1775764773.953307:0:19896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c7b5 from it ffffafbac1ec7cb8 00010000:00010000:2.0:1775764773.953310:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c7b5 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x267:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fb05a expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764773.953318:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c7b5 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x267:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fb05a expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764773.953343:0:19896:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x267:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1775764773.953352:0:19896:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0x267:0x0] 00000008:00010000:2.0:1775764773.953377:0:19896:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764773.953379:0:19896:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994166f7b0, osc lock ffff8f994f1a2ab0, flags 0x400001002 00010000:00010000:2.0:1775764773.953394:0:19896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000031070867/0x36f9bdc9e421c164 lrc: 2/1,0 mode: PR/PR res: [0x280000403:0x33:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5f888a expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764773.953412:0:19896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000031070867/0x36f9bdc9e421c164 lrc: 2/1,0 mode: PR/PR res: [0x280000403:0x33:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5f888a expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:2.0:1775764773.953424:0:19896:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1775764773.953427:0:19896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000031070867/0x36f9bdc9e421c164 lrc: 5/1,0 mode: PR/PR res: [0x280000403:0x33:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5f888a expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764773.953436:0:19896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000031070867/0x36f9bdc9e421c164 lrc: 4/0,0 mode: PR/PR res: [0x280000403:0x33:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5f888a expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:2.0:1775764773.953445:0:19896:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1775764773.953453:0:19896:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0x267:0x0] updating i_size 1099872 i_blocks 4103 00000020:00010000:2.0:1775764773.953458:0:19896:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f997045a6c8 (ffff8f994ef58a28/0) at cl_lock_release():245 00000020:00010000:2.0:1775764773.953461:0:19896:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f997045a6c8 (ffff8f994ef58a28/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764773.953464:0:19896:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994166f7b0 (ffff8f994ef585a8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764773.953468:0:19896:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f997045a6c8 (ffff8f994ef58a28/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764773.953471:0:19896:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994166f7b0 (ffff8f994ef58a28/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764773.953490:0:19896:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x267:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764773.953524:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:2.0:1775764773.953531:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=lib64, flags=16 00000080:00200000:2.0:1775764773.953536:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x264:0x0](ffff8f9952da1988), inode mode 41ed mask 1 00000080:00200000:2.0:1775764773.953541:0:19896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=ld-linux-x86-64.so.2, flags=0 00000080:00200000:2.0:1775764773.953544:0:19896:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x264:0x0](ffff8f9952da1988), inode mode 41ed mask 3 00000080:00200000:2.0:1775764773.953549:0:19896:0:(namei.c:2395:ll_unlink()) VFS Op:name=ld-linux-x86-64.so.2, dir=[0x200000401:0x264:0x0](ffff8f9952da1988) 00010000:00010000:2.0:1775764773.953586:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c7b5 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x267:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5fb05a expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764773.953622:0:19896:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x267:0x0]: losing layout lock 00010000:00010000:2.0:1775764773.953641:0:19896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d8400/0x36f9bdc9e421c15d lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x267:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5f8883 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1775764773.953655:0:19896:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0x267:0x0]: losing layout lock 00010000:00010000:2.0:1775764773.953687:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c7b5 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x267:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fb05a expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764773.953694:0:19896:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d8400/0x36f9bdc9e421c15d lrc: 1/0,0 mode: --/CR res: [0x200000401:0x267:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f8883 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764773.953700:0:19896:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:2.0:1775764773.953702:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c7b5 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x267:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fb05a expref: -99 pid: 19896 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764773.953715:0:19896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f994f3d8400/0x36f9bdc9e421c15d lrc: 0/0,0 mode: --/CR res: [0x200000401:0x267:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5f8883 expref: -99 pid: 19724 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000020:00100000:2.0:1775764773.953788:0:19896:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1775764773.953797:0:19896:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994efdc700 x1862022967009152/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 504/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764773.953820:0:19896:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdc700 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967009152:192.168.202.146@tcp:36:rm.0 00000100:00100000:2.0:1775764773.953891:0:19896:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.969368:0:19896:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdc700 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19896:1862022967009152:192.168.202.146@tcp:36:rm.0 00000080:00200000:2.0:1775764773.969454:0:19896:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0x267:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764773.969468:0:19896:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:2.0:1775764773.969490:0:19896:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:2.0:1775764773.969492:0:19896:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:2.0:1775764773.969529:0:19896:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0x267:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764773.975724:0:19896:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x267:0x0](ffff8f9952e63248) 00000008:00010000:2.0:1775764773.975819:0:19896:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 1099872 blocks 4103 [cma]time: 1775764048 1749497576 0 by oinfo size 1099872 blocks 4103 [cma]time 1775764048 1749497576 0 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000031070867/0x36f9bdc9e421c164 lrc: 1/0,0 mode: PR/PR res: [0x280000403:0x33:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5f888a expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000080:00200008:3.0:1775764773.978949:0:19698:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8f9952d121c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764773.978960:0:19698:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:3.0:1775764773.978983:0:19698:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764773.979000:0:19698:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1775764773.979016:0:19698:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1775764773.979024:0:19698:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764773.979032:0:19698:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:3.0:1775764773.979035:0:19698:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c67a 00000080:00010000:3.0:1775764773.979043:0:19698:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c67a from it ffffafbac19c7cb8 00010000:00010000:3.0:1775764773.979046:0:19698:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1775764773.979056:0:19698:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1775764773.979082:0:19698:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:3.0:1775764773.979088:0:19698:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764773.979093:0:19698:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1775764773.979102:0:19698:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1775764773.979108:0:19698:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764773.979112:0:19698:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:3.0:1775764773.979115:0:19698:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c67a 00000080:00010000:3.0:1775764773.979118:0:19698:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c67a from it ffffafbac19c7b58 00010000:00010000:3.0:1775764773.979121:0:19698:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1775764773.979128:0:19698:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:3.0:1775764773.979136:0:19698:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:3.0:1775764773.979151:0:19698:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=d8.sanity-pfl, dir=[0x200000007:0x1:0x0](ffff8f9952d121c8), flags=9 00000080:00200000:3.0:1775764773.979155:0:19698:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=d8.sanity-pfl, dir=[0x200000007:0x1:0x0](ffff8f9952d121c8), intent=getattr 00000002:00010000:3.0:1775764773.979165:0:19698:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: d8.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764773.979206:0:19698:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c7bc lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19698 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.979214:0:19698:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c7bc lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19698 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.979224:0:19698:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c7bc lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19698 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764773.979252:0:19698:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job bash:3f83546c-01a9-4156-8a97-44c446245d3e:19698:1862022967009280:192.168.202.146@tcp:101:bash.0 00000100:00100000:3.0:1775764773.979353:0:19698:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99524133c0 going to sleep for 11 seconds 00000100:00100000:3.0:1775764773.989394:0:19698:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job bash:3f83546c-01a9-4156-8a97-44c446245d3e:19698:1862022967009280:192.168.202.146@tcp:101:bash.0 00010000:00010000:3.0:1775764773.989427:0:19698:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c7bc lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb092 expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.989449:0:19698:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c7bc lrc: 4/1,0 mode: --/PR res: [0x200000401:0x261:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb092 expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.989458:0:19698:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c7bc lrc: 4/1,0 mode: --/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb092 expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.989471:0:19698:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c7bc lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb092 expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.989485:0:19698:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c7bc lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb092 expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.989516:0:19698:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c7bc lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb092 expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.989527:0:19698:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c7bc lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb092 expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.989536:0:19698:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c7bc lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb092 expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764773.989549:0:19698:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970edf100 x1862022967009280/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/584 e 0 to 0 dl 1775764789 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1775764773.989566:0:19698:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c7bc lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb092 expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.989578:0:19698:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/609 (0 0) 00000080:00200000:3.0:1775764773.989586:0:19698:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:3.0:1775764773.989626:0:19698:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x261:0x0], updating i_size 7680 00000080:00200000:3.0:1775764773.989630:0:19698:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x261:0x0](ffff8f9952e61988): rc = 0 00000080:00010000:3.0:1775764773.989636:0:19698:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x261:0x0](ffff8f9952e61988) for lock 0x36f9bdc9e421c7bc 00000080:00010000:3.0:1775764773.989658:0:19698:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x261:0x0](ffff8f9952e61988) 00000080:00010000:3.0:1775764773.989662:0:19698:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c7bc from it ffffafbac19c7b20 00010000:00010000:3.0:1775764773.989665:0:19698:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c7bc lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb092 expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764773.989674:0:19698:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c7bc lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb092 expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:3.0:1775764773.989774:0:19698:0:(file.c:6226:ll_getattr_dentry()) START file d8.sanity-pfl:[0x200000401:0x261:0x0](ffff8f9952e61988), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764773.989782:0:19698:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988),name=d8.sanity-pfl 00000002:00010000:3.0:1775764773.989802:0:19698:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x261:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764773.989812:0:19698:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c7bc lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb092 expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764773.989822:0:19698:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c7bc lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb092 expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764773.989828:0:19698:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764773.989832:0:19698:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x261:0x0](ffff8f9952e61988) 00000080:00010000:3.0:1775764773.989835:0:19698:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x261:0x0](ffff8f9952e61988) for lock 0x36f9bdc9e421c7bc 00000080:00010000:3.0:1775764773.989839:0:19698:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c7bc from it ffffafbac19c7cb8 00010000:00010000:3.0:1775764773.989842:0:19698:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c7bc lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb092 expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764773.989848:0:19698:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c7bc lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb092 expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764773.989884:0:19698:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:3.0:1775764773.989890:0:19698:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764773.989896:0:19698:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:3.0:1775764773.989905:0:19698:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:3.0:1775764773.989911:0:19698:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764773.989921:0:19698:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:3.0:1775764773.989923:0:19698:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c67a 00000080:00010000:3.0:1775764773.989926:0:19698:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c67a from it ffffafbac19c7c18 00010000:00010000:3.0:1775764773.989929:0:19698:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:3.0:1775764773.989935:0:19698:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:3.0:1775764773.989942:0:19698:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:3.0:1775764773.989948:0:19698:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d8.sanity-pfl, flags=11 00000080:00200000:3.0:1775764773.989954:0:19698:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 101 00000080:00200000:1.0:1775764774.007811:0:19898:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764774.007840:0:19898:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764774.007858:0:19898:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764774.007874:0:19898:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1775764774.007888:0:19898:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764774.007896:0:19898:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764774.007900:0:19898:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c67a 00000080:00010000:1.0:1775764774.007906:0:19898:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c67a from it ffffafbac1ec7b58 00010000:00010000:1.0:1775764774.007910:0:19898:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764774.007920:0:19898:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1775764774.007934:0:19898:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764774.007941:0:19898:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d8.sanity-pfl, flags=8 00000080:00200008:1.0:1775764774.007950:0:19898:0:(file.c:6226:ll_getattr_dentry()) START file d8.sanity-pfl:[0x200000401:0x261:0x0](ffff8f9952e61988), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764774.007954:0:19898:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988),name=d8.sanity-pfl 00000002:00010000:1.0:1775764774.007965:0:19898:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x261:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764774.007973:0:19898:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c7bc lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb092 expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764774.007981:0:19898:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c7bc lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb092 expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764774.007987:0:19898:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764774.007991:0:19898:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x261:0x0](ffff8f9952e61988) 00000080:00010000:1.0:1775764774.007993:0:19898:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x261:0x0](ffff8f9952e61988) for lock 0x36f9bdc9e421c7bc 00000080:00010000:1.0:1775764774.007997:0:19898:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c7bc from it ffffafbac1ec7cb8 00010000:00010000:1.0:1775764774.007999:0:19898:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c7bc lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb092 expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764774.008007:0:19898:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c7bc lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb092 expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764774.008035:0:19898:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764774.008041:0:19898:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764774.008046:0:19898:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764774.008054:0:19898:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1775764774.008060:0:19898:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764774.008064:0:19898:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764774.008066:0:19898:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c67a 00000080:00010000:1.0:1775764774.008069:0:19898:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c67a from it ffffafbac1ec7bf8 00010000:00010000:1.0:1775764774.008072:0:19898:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764774.008078:0:19898:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:1.0:1775764774.008086:0:19898:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764774.008089:0:19898:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d8.sanity-pfl, flags=266 00000080:00200000:1.0:1775764774.008092:0:19898:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 44 00000080:00200008:1.0:1775764774.008097:0:19898:0:(file.c:1057:ll_file_open()) START file d8.sanity-pfl:[0x200000401:0x261:0x0](ffff8f9952e61988), flags 704400 00000002:00010000:1.0:1775764774.008111:0:19898:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x261:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100240704401 00010000:00010000:1.0:1775764774.008163:0:19898:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c7c3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19898 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764774.008170:0:19898:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c7c3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19898 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764774.008181:0:19898:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764774.008193:0:19898:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e7100 x1862022967009536/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/135824 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764774.008203:0:19898:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c7c3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19898 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764774.008225:0:19898:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7100 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19898:1862022967009536:192.168.202.146@tcp:101:rm.0 00000100:00100000:1.0:1775764774.015733:0:19898:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f997130d780 going to sleep for 11 seconds 00000100:00100000:2.0:1775764774.025300:0:19898:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7100 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19898:1862022967009536:192.168.202.146@tcp:101:rm.0 00010000:00010000:2.0:1775764774.025324:0:19898:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c7c3 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19898 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.025339:0:19898:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c7c3 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 19898 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.025347:0:19898:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c7c3 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 19898 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.025367:0:19898:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c7c3 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 19898 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.025376:0:19898:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c7c3 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 19898 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.025382:0:19898:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970640c00 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:2.0:1775764774.025387:0:19898:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c7c3 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 19898 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.025394:0:19898:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c7c3 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 19898 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.025406:0:19898:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c7c3 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 19898 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.025413:0:19898:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1775764774.025419:0:19898:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c7c3 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 19898 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.025426:0:19898:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c7c3 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 19898 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764774.025446:0:19898:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8f994f2e7100 x1862022967009536/t4294997589(4294997589) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764790 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764774.025460:0:19898:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f994f2e7100 x1862022967009536/t4294997589(4294997589) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764790 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764774.025473:0:19898:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764774.025503:0:19898:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x261:0x0], updating i_size 7680 00000002:00100000:2.0:1775764774.025524:0:19898:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f994f2e7100 x1862022967009536/t4294997589(4294997589) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764790 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200008:2.0:1775764774.025610:0:19898:0:(file.c:6226:ll_getattr_dentry()) START file d8.sanity-pfl:[0x200000401:0x261:0x0](ffff8f9952e61988), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764774.025617:0:19898:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988),name=d8.sanity-pfl 00000002:00010000:2.0:1775764774.025636:0:19898:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x261:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764774.025649:0:19898:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c7bc lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb092 expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764774.025660:0:19898:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c7bc lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb092 expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764774.025673:0:19898:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764774.025678:0:19898:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x261:0x0](ffff8f9952e61988) 00000080:00010000:2.0:1775764774.025682:0:19898:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x261:0x0](ffff8f9952e61988) for lock 0x36f9bdc9e421c7bc 00000080:00010000:2.0:1775764774.025686:0:19898:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c7bc from it ffffafbac1ec7cd8 00010000:00010000:2.0:1775764774.025690:0:19898:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c7bc lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb092 expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764774.025697:0:19898:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c7bc lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb092 expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1775764774.025815:0:19898:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988) pos/size0/7680 32bit_api 0 00000002:00010000:2.0:1775764774.025829:0:19898:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x261:0x0]) in obj [0x200000401:0x261:0x0], intent: readdir flags 0 00010000:00010000:2.0:1775764774.025837:0:19898:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c7bc lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb092 expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764774.025844:0:19898:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c7bc lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb092 expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000100:00100000:2.0:1775764774.026468:0:19898:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdf480 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19898:1862022967009664:192.168.202.146@tcp:37:rm.0 00000100:00100000:2.0:1775764774.026561:0:19898:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102480 going to sleep for 11 seconds 00000100:00100000:2.0:1775764774.036675:0:19898:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdf480 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19898:1862022967009664:192.168.202.146@tcp:37:rm.0 00010000:00010000:2.0:1775764774.038083:0:19898:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c7bc lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb092 expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764774.038102:0:19898:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c7bc lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb092 expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200008:2.0:1775764774.038249:0:19898:0:(file.c:447:ll_file_release()) START file d8.sanity-pfl:[0x200000401:0x261:0x0](ffff8f9952e61988), flags 704000 00000002:00100000:2.0:1775764774.038316:0:19898:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f994f2e7100 x1862022967009536/t4294997589(4294997589) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764790 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'rm.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764774.038328:0:19898:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764774.038334:0:19898:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994efdf480 x1862022967009920/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764774.038374:0:19898:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdf480 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19898:1862022967009920:192.168.202.146@tcp:35:rm.0 00000100:00100000:2.0:1775764774.038480:0:19898:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102480 going to sleep for 11 seconds 00000100:00100000:2.0:1775764774.045302:0:19898:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdf480 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19898:1862022967009920:192.168.202.146@tcp:35:rm.0 00000002:00100000:2.0:1775764774.045334:0:19898:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f994f2e7100 x1862022967009536/t4294997589(4294997589) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764790 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764774.045425:0:19898:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:2.0:1775764774.045443:0:19898:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764774.045461:0:19898:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:2.0:1775764774.045473:0:19898:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:2.0:1775764774.045480:0:19898:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764774.045486:0:19898:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:2.0:1775764774.045489:0:19898:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c67a 00000080:00010000:2.0:1775764774.045496:0:19898:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c67a from it ffffafbac1ec7bf8 00010000:00010000:2.0:1775764774.045500:0:19898:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:2.0:1775764774.045509:0:19898:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:2.0:1775764774.045523:0:19898:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:2.0:1775764774.045531:0:19898:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d8.sanity-pfl, flags=266 00000080:00200000:2.0:1775764774.045538:0:19898:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 44 00000080:00200008:2.0:1775764774.045546:0:19898:0:(file.c:1057:ll_file_open()) START file d8.sanity-pfl:[0x200000401:0x261:0x0](ffff8f9952e61988), flags 704400 00000002:00010000:2.0:1775764774.045561:0:19898:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x261:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0104240704401 00010000:00010000:2.0:1775764774.045617:0:19898:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c7ca lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19898 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.045625:0:19898:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c7ca lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19898 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764774.045635:0:19898:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764774.045640:0:19898:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994efdf480 x1862022967010176/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/135824 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764774.045649:0:19898:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c7ca lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19898 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764774.045670:0:19898:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994efdf480 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19898:1862022967010176:192.168.202.146@tcp:101:rm.0 00000100:00100000:2.0:1775764774.045870:0:19898:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102480 going to sleep for 11 seconds 00000100:00100000:2.0:1775764774.050268:0:19898:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994efdf480 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19898:1862022967010176:192.168.202.146@tcp:101:rm.0 00010000:00010000:2.0:1775764774.050301:0:19898:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c7ca lrc: 4/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb0ae expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.050316:0:19898:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c7ca lrc: 4/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb0ae expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.050328:0:19898:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c7ca lrc: 4/1,0 mode: CR/CR res: [0x200000401:0x261:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb0ae expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.050338:0:19898:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c7ca lrc: 4/1,0 mode: CR/CR res: [0x200000401:0x261:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb0ae expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764774.050353:0:19898:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=200002b, status=0 req@ffff8f994efdf480 x1862022967010176/t4294997591(4294997591) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764790 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764774.050378:0:19898:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f994efdf480 x1862022967010176/t4294997591(4294997591) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764790 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764774.050389:0:19898:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c7ca lrc: 3/1,0 mode: CR/CR res: [0x200000401:0x261:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb0ae expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.050399:0:19898:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 16 res 8589935617/609 (0 0) 00000080:00200000:2.0:1775764774.050407:0:19898:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764774.050430:0:19898:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x261:0x0], updating i_size 7680 00000080:00010000:2.0:1775764774.050436:0:19898:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x261:0x0](ffff8f9952e61988) for lock 0x36f9bdc9e421c7ca 00000080:00010000:2.0:1775764774.050446:0:19898:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c7ca from it ffffafbac1ec7bd0 00010000:00010000:2.0:1775764774.050451:0:19898:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c7ca lrc: 3/1,0 mode: CR/CR res: [0x200000401:0x261:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb0ae expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.050461:0:19898:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c7ca lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x261:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb0ae expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000002:00100000:2.0:1775764774.050493:0:19898:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f994efdf480 x1862022967010176/t4294997591(4294997591) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764790 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200008:2.0:1775764774.050575:0:19898:0:(file.c:6226:ll_getattr_dentry()) START file d8.sanity-pfl:[0x200000401:0x261:0x0](ffff8f9952e61988), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764774.050582:0:19898:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988),name=d8.sanity-pfl 00000002:00010000:2.0:1775764774.050613:0:19898:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x261:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764774.050626:0:19898:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c7bc lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb092 expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764774.050636:0:19898:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c7bc lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb092 expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1775764774.050644:0:19898:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764774.050649:0:19898:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x261:0x0](ffff8f9952e61988) 00000080:00010000:2.0:1775764774.050653:0:19898:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x261:0x0](ffff8f9952e61988) for lock 0x36f9bdc9e421c7bc 00000080:00010000:2.0:1775764774.050656:0:19898:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c7bc from it ffffafbac1ec7cd8 00010000:00010000:2.0:1775764774.050659:0:19898:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c7bc lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb092 expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764774.050667:0:19898:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c7bc lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb092 expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:2.0:1775764774.050788:0:19898:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988) pos/size0/7680 32bit_api 0 00000080:00200000:2.0:1775764774.050815:0:19898:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988) pos/size18446744073709551614/7680 32bit_api 0 00000080:00200000:2.0:1775764774.050837:0:19898:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:2.0:1775764774.050844:0:19898:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=lib64, flags=0 00000080:00200008:2.0:1775764774.050852:0:19898:0:(file.c:6226:ll_getattr_dentry()) START file lib64:[0x200000401:0x264:0x0](ffff8f9952da1988), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764774.052518:0:19898:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x264:0x0](ffff8f9952da1988),name=lib64 00000002:00010000:2.0:1775764774.052538:0:19898:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x264:0x0]) in obj [0x200000401:0x261:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764774.052549:0:19898:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/612 (0 0) 00010000:00010000:2.0:1775764774.052608:0:19898:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c7d1 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 19898 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.052626:0:19898:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c7d1 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19898 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.052637:0:19898:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c7d1 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19898 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764774.052660:0:19898:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19898:1862022967010304:192.168.202.146@tcp:101:rm.0 00000100:00100000:2.0:1775764774.052791:0:19898:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99601023c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764774.063669:0:19898:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19898:1862022967010304:192.168.202.146@tcp:101:rm.0 00010000:00010000:2.0:1775764774.063696:0:19898:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c7d1 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb0c3 expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.063714:0:19898:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c7d1 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x264:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb0c3 expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.063722:0:19898:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c7d1 lrc: 4/1,0 mode: --/PR res: [0x200000401:0x264:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb0c3 expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.063734:0:19898:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c7d1 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x264:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb0c3 expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.063781:0:19898:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c7d1 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x264:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb0c3 expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.063794:0:19898:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c7d1 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x264:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb0c3 expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.063803:0:19898:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c7d1 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x264:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb0c3 expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.063810:0:19898:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c7d1 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x264:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb0c3 expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764774.063821:0:19898:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f99703d0e00 x1862022967010304/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/584 e 0 to 0 dl 1775764790 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764774.063833:0:19898:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c7d1 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x264:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb0c3 expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.063843:0:19898:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/612 (0 0) 00000080:00200000:2.0:1775764774.063849:0:19898:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x261:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764774.063871:0:19898:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x264:0x0], updating i_size 7168 00000080:00010000:2.0:1775764774.063876:0:19898:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x264:0x0](ffff8f9952da1988) for lock 0x36f9bdc9e421c7d1 00000080:00010000:2.0:1775764774.063885:0:19898:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x264:0x0](ffff8f9952da1988) 00000080:00010000:2.0:1775764774.063888:0:19898:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c7d1 from it ffffafbac1ec7cb8 00010000:00010000:2.0:1775764774.063890:0:19898:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c7d1 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x264:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb0c3 expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.063898:0:19898:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c7d1 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x264:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb0c3 expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764774.064007:0:19898:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:2.0:1775764774.064017:0:19898:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=lib64, flags=258 00000080:00200000:2.0:1775764774.064040:0:19898:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x264:0x0](ffff8f9952da1988), inode mode 41ed mask 44 00000080:00200008:2.0:1775764774.064050:0:19898:0:(file.c:1057:ll_file_open()) START file lib64:[0x200000401:0x264:0x0](ffff8f9952da1988), flags 704400 00000002:00010000:2.0:1775764774.064072:0:19898:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x264:0x0]) in obj [0x200000401:0x261:0x0], intent: open flags 0100240704401 00010000:00010000:2.0:1775764774.064127:0:19898:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c7d8 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 19898 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.064135:0:19898:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c7d8 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19898 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764774.064146:0:19898:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764774.064153:0:19898:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f99703d0e00 x1862022967010560/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/135824 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764774.064162:0:19898:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c7d8 lrc: 3/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19898 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764774.064186:0:19898:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19898:1862022967010560:192.168.202.146@tcp:101:rm.0 00000100:00100000:2.0:1775764774.064348:0:19898:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99601023c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764774.082268:0:19898:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19898:1862022967010560:192.168.202.146@tcp:101:rm.0 00010000:00010000:2.0:1775764774.082301:0:19898:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c7d8 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 19898 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.082314:0:19898:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c7d8 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 19898 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.082322:0:19898:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c7d8 lrc: 4/1,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 19898 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.082329:0:19898:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c7d8 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 19898 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.082338:0:19898:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c7d8 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 19898 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.082344:0:19898:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99706eba00 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:2.0:1775764774.082348:0:19898:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c7d8 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 19898 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.082365:0:19898:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c7d8 lrc: 4/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 19898 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.082372:0:19898:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c7d8 lrc: 5/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 19898 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.082379:0:19898:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1775764774.082386:0:19898:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c7d8 lrc: 3/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 19898 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.082392:0:19898:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eba00/0x36f9bdc9e421c7d8 lrc: 0/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 19898 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764774.082412:0:19898:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8f99703d0e00 x1862022967010560/t4294997592(4294997592) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764790 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764774.082425:0:19898:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f99703d0e00 x1862022967010560/t4294997592(4294997592) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764790 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764774.082439:0:19898:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0x261:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764774.082458:0:19898:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x264:0x0], updating i_size 7168 00000002:00100000:2.0:1775764774.082478:0:19898:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f99703d0e00 x1862022967010560/t4294997592(4294997592) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764790 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200008:2.0:1775764774.082550:0:19898:0:(file.c:6226:ll_getattr_dentry()) START file lib64:[0x200000401:0x264:0x0](ffff8f9952da1988), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764774.082557:0:19898:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0x264:0x0](ffff8f9952da1988),name=lib64 00000002:00010000:2.0:1775764774.082577:0:19898:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x264:0x0]) in obj [0x200000401:0x261:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764774.082601:0:19898:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c7d1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x264:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb0c3 expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764774.082612:0:19898:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c7d1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x264:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb0c3 expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764774.082620:0:19898:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0x261:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764774.082625:0:19898:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x264:0x0](ffff8f9952da1988) 00000080:00010000:2.0:1775764774.082629:0:19898:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x264:0x0](ffff8f9952da1988) for lock 0x36f9bdc9e421c7d1 00000080:00010000:2.0:1775764774.082634:0:19898:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c7d1 from it ffffafbac1ec7cd8 00010000:00010000:2.0:1775764774.082639:0:19898:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c7d1 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x264:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb0c3 expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764774.082647:0:19898:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c7d1 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x264:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb0c3 expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764774.082679:0:19898:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x264:0x0](ffff8f9952da1988) pos/size0/7168 32bit_api 0 00000002:00010000:2.0:1775764774.082691:0:19898:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0x264:0x0]) in obj [0x200000401:0x264:0x0], intent: readdir flags 0 00010000:00010000:2.0:1775764774.082699:0:19898:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c7d1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x264:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb0c3 expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764774.082706:0:19898:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c7d1 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0x264:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb0c3 expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000100:00100000:2.0:1775764774.083605:0:19898:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19898:1862022967010688:192.168.202.146@tcp:37:rm.0 00000100:00100000:2.0:1775764774.083706:0:19898:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99601023c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764774.111970:0:19898:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19898:1862022967010688:192.168.202.146@tcp:37:rm.0 00010000:00010000:2.0:1775764774.112603:0:19898:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c7d1 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x264:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb0c3 expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764774.112619:0:19898:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c7d1 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x264:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb0c3 expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764774.112668:0:19898:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200000401:0x264:0x0](ffff8f9952da1988) pos/size18446744073709551614/7168 32bit_api 0 00000080:00200008:2.0:1775764774.112687:0:19898:0:(file.c:447:ll_file_release()) START file lib64:[0x200000401:0x264:0x0](ffff8f9952da1988), flags 704000 00000002:00100000:2.0:1775764774.112775:0:19898:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f99703d0e00 x1862022967010560/t4294997592(4294997592) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764790 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'rm.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764774.112787:0:19898:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764774.112793:0:19898:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f99703d0700 x1862022967010944/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764774.112814:0:19898:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19898:1862022967010944:192.168.202.146@tcp:35:rm.0 00000100:00100000:2.0:1775764774.113098:0:19898:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99601023c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764774.121822:0:19898:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19898:1862022967010944:192.168.202.146@tcp:35:rm.0 00000002:00100000:2.0:1775764774.121855:0:19898:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f99703d0e00 x1862022967010560/t4294997592(4294997592) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764790 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764774.121925:0:19898:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 1 00000080:00200000:2.0:1775764774.121939:0:19898:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=lib64, flags=0 00000080:00200000:2.0:1775764774.121948:0:19898:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988), inode mode 41ed mask 3 00000080:00200000:2.0:1775764774.121954:0:19898:0:(namei.c:2305:ll_rmdir()) VFS Op:name=lib64, dir=[0x200000401:0x261:0x0](ffff8f9952e61988) 00010000:00010000:2.0:1775764774.121987:0:19898:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c7bc lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fb092 expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764774.122010:0:19898:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c7ca lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x261:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb0ae expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764774.122034:0:19898:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c7d1 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x264:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fb0c3 expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764774.122098:0:19898:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c799 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x264:0x0].0x0 bits 0x11/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5faff8 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764774.122135:0:19898:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c7d1 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x264:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fb0c3 expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764774.122142:0:19898:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c799 lrc: 1/0,0 mode: --/PR res: [0x200000401:0x264:0x0].0x0 bits 0x11/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5faff8 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764774.122148:0:19898:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c7bc lrc: 1/0,0 mode: --/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fb092 expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764774.122153:0:19898:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:2.0:1775764774.122156:0:19898:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eaa00/0x36f9bdc9e421c7d1 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x264:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fb0c3 expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764774.122165:0:19898:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c799 lrc: 0/0,0 mode: --/PR res: [0x200000401:0x264:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5faff8 expref: -99 pid: 19896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764774.122182:0:19898:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c7bc lrc: 0/0,0 mode: --/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fb092 expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000020:00100000:2.0:1775764774.122195:0:19898:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1775764774.122201:0:19898:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f99703d0700 x1862022967011200/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 496/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764774.122223:0:19898:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19898:1862022967011200:192.168.202.146@tcp:36:rm.0 00000100:00100000:2.0:1775764774.122315:0:19898:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99601023c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764774.127949:0:19898:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19898:1862022967011200:192.168.202.146@tcp:36:rm.0 00000080:00200000:2.0:1775764774.127996:0:19898:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x264:0x0](ffff8f9952da1988) 00000080:00200008:2.0:1775764774.128057:0:19898:0:(file.c:447:ll_file_release()) START file d8.sanity-pfl:[0x200000401:0x261:0x0](ffff8f9952e61988), flags 704000 00010000:00010000:2.0:1775764774.128094:0:19898:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c7ca lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x261:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb0ae expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764774.128131:0:19898:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:2.0:1775764774.128148:0:19898:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764774.128157:0:19898:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:2.0:1775764774.128165:0:19898:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:2.0:1775764774.128172:0:19898:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764774.128177:0:19898:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:2.0:1775764774.128180:0:19898:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c67a 00000080:00010000:2.0:1775764774.128187:0:19898:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c67a from it ffffafbac1ec7c10 00010000:00010000:2.0:1775764774.128191:0:19898:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:2.0:1775764774.128199:0:19898:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00000080:00200000:2.0:1775764774.128214:0:19898:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:2.0:1775764774.128229:0:19898:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=d8.sanity-pfl, dir=[0x200000007:0x1:0x0](ffff8f9952d121c8), flags=0 00000080:00200000:2.0:1775764774.128233:0:19898:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=d8.sanity-pfl, dir=[0x200000007:0x1:0x0](ffff8f9952d121c8), intent=getattr 00000002:00010000:2.0:1775764774.128245:0:19898:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: d8.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764774.128283:0:19898:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c7df lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 19898 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.128290:0:19898:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c7df lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19898 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.128300:0:19898:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c7df lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 19898 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764774.128318:0:19898:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19898:1862022967011328:192.168.202.146@tcp:101:rm.0 00000100:00100000:2.0:1775764774.128722:0:19898:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99601023c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764774.138242:0:19898:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19898:1862022967011328:192.168.202.146@tcp:101:rm.0 00010000:00010000:2.0:1775764774.138266:0:19898:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c7df lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb102 expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.138283:0:19898:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c7df lrc: 4/1,0 mode: --/PR res: [0x200000401:0x261:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb102 expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.138297:0:19898:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c7df lrc: 4/1,0 mode: --/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb102 expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.138308:0:19898:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c7df lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb102 expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.138319:0:19898:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c7df lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb102 expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.138331:0:19898:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c7df lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb102 expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.138339:0:19898:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c7df lrc: 5/2,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb102 expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.138346:0:19898:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c7df lrc: 4/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb102 expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764774.138357:0:19898:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f99703d0e00 x1862022967011328/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/584 e 0 to 0 dl 1775764790 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764774.138369:0:19898:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c7df lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb102 expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.138379:0:19898:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/609 (0 0) 00000080:00200000:2.0:1775764774.138385:0:19898:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764774.138407:0:19898:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0x261:0x0], updating i_size 7680 00000080:00200000:2.0:1775764774.138410:0:19898:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0x261:0x0](ffff8f9952e61988): rc = 0 00000080:00010000:2.0:1775764774.138415:0:19898:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0x261:0x0](ffff8f9952e61988) for lock 0x36f9bdc9e421c7df 00000080:00010000:2.0:1775764774.138426:0:19898:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0x261:0x0](ffff8f9952e61988) 00000080:00010000:2.0:1775764774.138429:0:19898:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c7df from it ffffafbac1ec7e00 00010000:00010000:2.0:1775764774.138432:0:19898:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c7df lrc: 3/1,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb102 expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.138441:0:19898:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c7df lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb102 expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764774.138483:0:19898:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:2.0:1775764774.138511:0:19898:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764774.138521:0:19898:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00010000:00010000:2.0:1775764774.138531:0:19898:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00000080:00200000:2.0:1775764774.138538:0:19898:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764774.138542:0:19898:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:2.0:1775764774.138546:0:19898:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c67a 00000080:00010000:2.0:1775764774.138549:0:19898:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c67a from it ffffafbac1ec7d88 00010000:00010000:2.0:1775764774.138552:0:19898:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00010000:00010000:2.0:1775764774.138558:0:19898:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 0 00000080:00200000:2.0:1775764774.138567:0:19898:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 3 00000080:00200000:2.0:1775764774.138574:0:19898:0:(namei.c:2305:ll_rmdir()) VFS Op:name=d8.sanity-pfl, dir=[0x200000007:0x1:0x0](ffff8f9952d121c8) 00010000:00010000:2.0:1775764774.138601:0:19898:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 0 00010000:00010000:2.0:1775764774.138633:0:19898:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c7df lrc: 2/0,0 mode: PR/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fb102 expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764774.138645:0:19898:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c7ca lrc: 2/0,0 mode: CR/CR res: [0x200000401:0x261:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fb0ae expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000002:00100000:2.0:1775764774.138685:0:19898:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f994efdf480 x1862022967010176/t4294997591(4294997591) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764790 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'rm.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764774.138696:0:19898:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764774.138700:0:19898:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f99703d0e00 x1862022967011584/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764774.138720:0:19898:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19898:1862022967011584:192.168.202.146@tcp:35:rm.0 00000100:00100000:2.0:1775764774.138844:0:19898:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99601023c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764774.143378:0:19898:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19898:1862022967011584:192.168.202.146@tcp:35:rm.0 00000002:00100000:2.0:1775764774.143407:0:19898:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f994efdf480 x1862022967010176/t4294997591(4294997591) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764790 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'rm.0' uid:0 gid:0 projid:0 00010000:00010000:2.0:1775764774.143477:0:19898:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c7df lrc: 1/0,0 mode: --/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fb102 expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764774.143487:0:19898:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c7ca lrc: 1/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fb0ae expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764774.143495:0:19898:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 0 00010000:00010000:2.0:1775764774.143509:0:19898:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:2.0:1775764774.143512:0:19898:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb600/0x36f9bdc9e421c7df lrc: 0/0,0 mode: --/PR res: [0x200000401:0x261:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fb102 expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764774.143523:0:19898:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb200/0x36f9bdc9e421c7ca lrc: 0/0,0 mode: --/CR res: [0x200000401:0x261:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fb0ae expref: -99 pid: 19898 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764774.143535:0:19898:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c67a lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fa86c expref: -99 pid: 19698 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 0 00000020:00100000:2.0:1775764774.143554:0:19898:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1775764774.143560:0:19898:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f99703d0e00 x1862022967011840/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 504/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764774.143582:0:19898:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19898:1862022967011840:192.168.202.146@tcp:36:rm.0 00000100:00100000:2.0:1775764774.143677:0:19898:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99601023c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764774.153055:0:19898:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:19898:1862022967011840:192.168.202.146@tcp:36:rm.0 00000080:00200000:2.0:1775764774.153102:0:19898:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0x261:0x0](ffff8f9952e61988) 00010000:00010000:3.0:1775764774.414438:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764774.600948:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2380:1862022967011968:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764774.601048:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967012224:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764774.601082:0:2380:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:0.0:1775764774.601089:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967012096:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764774.601115:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967012352:192.168.202.146@tcp:400:kworker.0 00000100:00100000:1.0:1775764774.618941:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967012096:192.168.202.146@tcp:400:kworker.0 00000100:00100000:1.0:1775764774.618957:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967012352:192.168.202.146@tcp:400:kworker.0 00000100:00100000:1.0:1775764774.618967:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2380:1862022967011968:192.168.202.146@tcp:400:kworker.0 00000100:00100000:1.0:1775764774.618975:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967012224:192.168.202.146@tcp:400:kworker.0 00010000:00010000:1.0:1775764775.435443:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764776.454462:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764777.482522:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764777.622078:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764777.622118:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764777.622144:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764777.622166:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764777.622208:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764777.622226:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764777.622253:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764777.622289:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764777.622330:0:5562:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1775764777.622348:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978386048 00000100:00100000:0.0:1775764777.622359:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978386048 00000100:00100000:0.0:1775764777.622438:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978386176 00000100:00100000:0.0:1775764777.622442:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978386176 00000100:00100000:0.0:1775764777.622454:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978386304 00000100:00100000:0.0:1775764777.622456:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978386304 00000100:00100000:0.0:1775764777.622469:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24186 00000100:00100000:0.0:1775764777.622478:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0ca300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+6:12051:x1862022978386048:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764777.622498:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f5ef7f23/0x36f9bdc9e421c1c6 lrc: 2/0,0 mode: PR/PR res: [0x280000403:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5f897f expref: -99 pid: 19850 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764777.622593:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764777.622600:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0ca300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+6:12051:x1862022978386048:12345-192.168.202.146@tcp:104: Request processed in 123us (537us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764777.622608:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24186 00000100:00100000:0.0:1775764777.622628:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24187 00000100:00100000:0.0:1775764777.622634:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c9c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:6578:x1862022978386176:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764777.622643:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000e275e2c0/0x36f9bdc9e420ed6c lrc: 2/0,0 mode: PR/PR res: [0x280000403:0x66:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5cd375 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00010000:00010000:0.0:1775764777.622678:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:0.0:1775764777.622683:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c9c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:6578:x1862022978386176:12345-192.168.202.146@tcp:104: Request processed in 49us (565us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764777.622689:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24187 00000100:00100000:0.0:1775764777.622700:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24188 00000100:00100000:0.0:1775764777.622706:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970e09450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:6579:x1862022978386304:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764777.622714:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000053f6e598/0x36f9bdc9e41cb08f lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x67:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef54f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 33 lru_type: 1 00010000:00010000:0.0:1775764777.622773:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:0.0:1775764777.622778:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970e09450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:6579:x1862022978386304:12345-192.168.202.146@tcp:104: Request processed in 73us (634us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764777.622783:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24188 00010000:00010000:0.0:1775764777.622823:0:19800:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:0.0:1775764777.622828:0:19800:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f5ef7f23/0x36f9bdc9e421c1c6 lrc: 2/0,0 mode: PR/PR res: [0x280000403:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5f897f expref: -99 pid: 19850 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764777.622836:0:19800:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99668b8000 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:0.0:1775764777.622842:0:19800:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f5ef7f23/0x36f9bdc9e421c1c6 lrc: 3/0,0 mode: PR/PR res: [0x280000403:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x428400010000 nid: local remote: 0x2aac6a50e5f897f expref: -99 pid: 19850 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764777.622928:0:19800:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f5ef7f23/0x36f9bdc9e421c1c6 lrc: 2/0,0 mode: --/PR res: [0x280000403:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5f897f expref: -99 pid: 19850 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764777.622937:0:19800:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764777.622958:0:19800:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f5ef7f23/0x36f9bdc9e421c1c6 lrc: 1/0,0 mode: --/PR res: [0x280000403:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5f897f expref: -99 pid: 19850 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764777.622966:0:19800:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f5ef7f23/0x36f9bdc9e421c1c6 lrc: 0/0,0 mode: --/PR res: [0x280000403:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5f897f expref: -99 pid: 19850 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764777.622992:0:19800:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1775764777.622996:0:19800:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000e275e2c0/0x36f9bdc9e420ed6c lrc: 2/0,0 mode: PR/PR res: [0x280000403:0x66:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5cd375 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00010000:00010000:0.0:1775764777.623002:0:19800:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99706ea000 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:0.0:1775764777.623005:0:19800:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000e275e2c0/0x36f9bdc9e420ed6c lrc: 3/0,0 mode: PR/PR res: [0x280000403:0x66:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x428400010000 nid: local remote: 0x2aac6a50e5cd375 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00010000:00010000:0.0:1775764777.623038:0:19800:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000e275e2c0/0x36f9bdc9e420ed6c lrc: 2/0,0 mode: --/PR res: [0x280000403:0x66:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5cd375 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00010000:00010000:0.0:1775764777.623044:0:19800:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764777.623050:0:19800:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000e275e2c0/0x36f9bdc9e420ed6c lrc: 1/0,0 mode: --/PR res: [0x280000403:0x66:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5cd375 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00010000:00010000:0.0:1775764777.623057:0:19800:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000e275e2c0/0x36f9bdc9e420ed6c lrc: 0/0,0 mode: --/PR res: [0x280000403:0x66:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5cd375 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00010000:00010000:0.0:1775764777.623072:0:19800:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1775764777.623075:0:19800:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000053f6e598/0x36f9bdc9e41cb08f lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x67:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef54f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 33 lru_type: 1 00010000:00010000:0.0:1775764777.623080:0:19800:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99710c8600 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:0.0:1775764777.623083:0:19800:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000053f6e598/0x36f9bdc9e41cb08f lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x67:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ef54f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 33 lru_type: 1 00010000:00010000:0.0:1775764777.623113:0:19800:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000053f6e598/0x36f9bdc9e41cb08f lrc: 2/0,0 mode: --/PW res: [0x280000403:0x67:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef54f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 33 lru_type: 1 00010000:00010000:0.0:1775764777.623120:0:19800:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764777.623124:0:19800:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000053f6e598/0x36f9bdc9e41cb08f lrc: 1/0,0 mode: --/PW res: [0x280000403:0x67:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef54f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 33 lru_type: 1 00010000:00010000:0.0:1775764777.623132:0:19800:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000053f6e598/0x36f9bdc9e41cb08f lrc: 0/0,0 mode: --/PW res: [0x280000403:0x67:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef54f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 33 lru_type: 1 00010000:00010000:0.0:1775764777.623146:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764777.623148:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764777.623149:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764777.623156:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764777.623161:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764777.623179:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967012608:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764777.623242:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:0.0:1775764777.623249:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967012480:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764777.623271:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967012736:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764777.623871:0:5563:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:3.0:1775764777.623887:0:5563:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978386432 00000100:00100000:3.0:1775764777.623893:0:5563:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978386432 00000100:00100000:3.0:1775764777.623932:0:5563:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978386560 00000100:00100000:3.0:1775764777.623935:0:5563:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978386560 00000100:00100000:3.0:1775764777.623944:0:5563:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978386688 00000100:00100000:3.0:1775764777.623947:0:5563:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978386688 00000100:00100000:3.0:1775764777.623956:0:5563:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978386816 00000100:00100000:3.0:1775764777.623958:0:5563:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978386816 00000100:00100000:3.0:1775764777.623966:0:5563:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978386944 00000100:00100000:3.0:1775764777.623968:0:5563:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978386944 00000100:00100000:3.0:1775764777.623982:0:5563:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 1121 00000100:00100000:3.0:1775764777.623989:0:5563:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c8e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+8:12041:x1862022978386432:12345-192.168.202.146@tcp:104: 00010000:00010000:3.0:1775764777.624002:0:5563:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b03c608/0x36f9bdc9e421c12c lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f87fe expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:3.0:1775764777.624067:0:5563:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1775764777.624073:0:5563:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c8e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+8:12041:x1862022978386432:12345-192.168.202.146@tcp:104: Request processed in 85us (1907us total) trans 0 rc 0/0 00000100:00100000:3.0:1775764777.624081:0:5563:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 1121 00000100:00100000:3.0:1775764777.624096:0:5563:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 1122 00000100:00100000:3.0:1775764777.624102:0:5563:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c8000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+7:8216:x1862022978386560:12345-192.168.202.146@tcp:104: 00010000:00010000:3.0:1775764777.624110:0:5563:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000d94da50a/0x36f9bdc9e421c1cd lrc: 2/0,0 mode: PR/PR res: [0x280000403:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5f8978 expref: -99 pid: 19850 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764777.624137:0:5563:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:3.0:1775764777.624142:0:5563:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c8000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+7:8216:x1862022978386560:12345-192.168.202.146@tcp:104: Request processed in 41us (1935us total) trans 0 rc 0/0 00000100:00100000:3.0:1775764777.624147:0:5563:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 1122 00000100:00100000:3.0:1775764777.624156:0:5563:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 1123 00000100:00100000:3.0:1775764777.624161:0:5563:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c9500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+6:12186:x1862022978386688:12345-192.168.202.146@tcp:104: 00010000:00010000:3.0:1775764777.624331:0:5563:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f444aa51/0x36f9bdc9e41cae97 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x60:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef055 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 139 lru_type: 1 00010000:00010000:3.0:1775764777.624359:0:5563:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:3.0:1775764777.624365:0:5563:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c9500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+6:12186:x1862022978386688:12345-192.168.202.146@tcp:104: Request processed in 204us (2137us total) trans 0 rc 0/0 00000100:00100000:3.0:1775764777.624381:0:5563:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 1123 00000100:00100000:3.0:1775764777.624394:0:5563:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 1124 00000100:00100000:3.0:1775764777.624400:0:5563:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c8380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+5:12039:x1862022978386816:12345-192.168.202.146@tcp:104: 00010000:00010000:3.0:1775764777.624682:0:5563:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000fb0a8ec1/0x36f9bdc9e421b2df lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x65b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f5cb5 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00010000:00010000:3.0:1775764777.624714:0:5563:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/4 locks to priority blp list, 4 blwis in pool 00000100:00100000:3.0:1775764777.624719:0:5563:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c8380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+5:12039:x1862022978386816:12345-192.168.202.146@tcp:104: Request processed in 320us (2465us total) trans 0 rc 0/0 00000100:00100000:3.0:1775764777.624725:0:5563:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 1124 00000100:00100000:3.0:1775764777.624771:0:5563:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 1125 00000100:00100000:3.0:1775764777.624778:0:5563:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0ce450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:6580:x1862022978386944:12345-192.168.202.146@tcp:104: 00010000:00010000:3.0:1775764777.624785:0:5563:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000acca1f81/0x36f9bdc9e41caffc lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x65:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef3ab expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 87 lru_type: 1 00010000:00010000:3.0:1775764777.624809:0:5563:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/5 locks to priority blp list, 5 blwis in pool 00000100:00100000:3.0:1775764777.624813:0:5563:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0ce450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:6580:x1862022978386944:12345-192.168.202.146@tcp:104: Request processed in 37us (2526us total) trans 0 rc 0/0 00000100:00100000:3.0:1775764777.624818:0:5563:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 1125 00010000:00010000:0.0:1775764777.627283:0:11138:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 4 total in blp. (4 blwis in pool) 00010000:00010000:0.0:1775764777.627297:0:11138:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b03c608/0x36f9bdc9e421c12c lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f87fe expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:0.0:1775764777.627310:0:11138:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99668bfe00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:0.0:1775764777.627318:0:11138:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b03c608/0x36f9bdc9e421c12c lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e5f87fe expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:0.0:1775764777.627919:0:11138:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b03c608/0x36f9bdc9e421c12c lrc: 2/0,0 mode: --/PW res: [0x280000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f87fe expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:0.0:1775764777.627930:0:11138:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764777.627965:0:11138:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b03c608/0x36f9bdc9e421c12c lrc: 1/0,0 mode: --/PW res: [0x280000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f87fe expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:0.0:1775764777.627974:0:11138:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006b03c608/0x36f9bdc9e421c12c lrc: 0/0,0 mode: --/PW res: [0x280000403:0x65f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f87fe expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:0.0:1775764777.627994:0:11138:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 3 total in blp. (3 blwis in pool) 00010000:00010000:0.0:1775764777.627998:0:11138:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000d94da50a/0x36f9bdc9e421c1cd lrc: 2/0,0 mode: PR/PR res: [0x280000403:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5f8978 expref: -99 pid: 19850 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764777.628004:0:11138:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99668b8c00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:0.0:1775764777.628007:0:11138:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000d94da50a/0x36f9bdc9e421c1cd lrc: 3/0,0 mode: PR/PR res: [0x280000403:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x428400010000 nid: local remote: 0x2aac6a50e5f8978 expref: -99 pid: 19850 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764777.632403:0:11138:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000d94da50a/0x36f9bdc9e421c1cd lrc: 2/0,0 mode: --/PR res: [0x280000403:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5f8978 expref: -99 pid: 19850 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764777.632422:0:11138:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764777.632445:0:11138:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000d94da50a/0x36f9bdc9e421c1cd lrc: 1/0,0 mode: --/PR res: [0x280000403:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5f8978 expref: -99 pid: 19850 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764777.632455:0:11138:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000d94da50a/0x36f9bdc9e421c1cd lrc: 0/0,0 mode: --/PR res: [0x280000403:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5f8978 expref: -99 pid: 19850 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764777.632488:0:11138:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:0.0:1775764777.632494:0:11138:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f444aa51/0x36f9bdc9e41cae97 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x60:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef055 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 139 lru_type: 1 00010000:00010000:0.0:1775764777.632502:0:11138:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9966c08200 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:0.0:1775764777.632508:0:11138:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f444aa51/0x36f9bdc9e41cae97 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x60:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ef055 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 139 lru_type: 1 00010000:00010000:3.0:1775764777.632805:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1775764777.632813:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000fb0a8ec1/0x36f9bdc9e421b2df lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x65b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f5cb5 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00010000:00010000:3.0:1775764777.632821:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970972800 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:3.0:1775764777.632825:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000fb0a8ec1/0x36f9bdc9e421b2df lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x65b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e5f5cb5 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00010000:00010000:3.0:1775764777.632907:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000fb0a8ec1/0x36f9bdc9e421b2df lrc: 2/0,0 mode: --/PW res: [0x280000403:0x65b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f5cb5 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00010000:00010000:3.0:1775764777.632914:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764777.632935:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000fb0a8ec1/0x36f9bdc9e421b2df lrc: 1/0,0 mode: --/PW res: [0x280000403:0x65b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f5cb5 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00010000:00010000:3.0:1775764777.632942:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000fb0a8ec1/0x36f9bdc9e421b2df lrc: 0/0,0 mode: --/PW res: [0x280000403:0x65b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f5cb5 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00010000:00010000:3.0:1775764777.632957:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1775764777.632961:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000acca1f81/0x36f9bdc9e41caffc lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x65:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef3ab expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 87 lru_type: 1 00010000:00010000:3.0:1775764777.632968:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9941480800 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:3.0:1775764777.632972:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000acca1f81/0x36f9bdc9e41caffc lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x65:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ef3ab expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 87 lru_type: 1 00010000:00010000:3.0:1775764777.633002:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000acca1f81/0x36f9bdc9e41caffc lrc: 2/0,0 mode: --/PW res: [0x280000403:0x65:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef3ab expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 87 lru_type: 1 00010000:00010000:3.0:1775764777.633008:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764777.633014:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000acca1f81/0x36f9bdc9e41caffc lrc: 1/0,0 mode: --/PW res: [0x280000403:0x65:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef3ab expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 87 lru_type: 1 00010000:00010000:3.0:1775764777.633022:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000acca1f81/0x36f9bdc9e41caffc lrc: 0/0,0 mode: --/PW res: [0x280000403:0x65:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef3ab expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 87 lru_type: 1 00010000:00010000:3.0:1775764777.633035:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764777.633037:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764777.633039:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764777.633059:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764777.633095:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967013120:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764777.633206:0:2381:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1775764777.633213:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967013248:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764777.633313:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967012992:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764777.633349:0:2380:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1775764777.633355:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967012864:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1775764777.635871:0:11138:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f444aa51/0x36f9bdc9e41cae97 lrc: 2/0,0 mode: --/PW res: [0x280000403:0x60:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef055 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 139 lru_type: 1 00010000:00010000:0.0:1775764777.635888:0:11138:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764777.635909:0:11138:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f444aa51/0x36f9bdc9e41cae97 lrc: 1/0,0 mode: --/PW res: [0x280000403:0x60:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef055 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 139 lru_type: 1 00010000:00010000:0.0:1775764777.635918:0:11138:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f444aa51/0x36f9bdc9e41cae97 lrc: 0/0,0 mode: --/PW res: [0x280000403:0x60:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef055 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 139 lru_type: 1 00010000:00010000:0.0:1775764777.635947:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764777.635949:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764777.635951:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764777.636153:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764777.636162:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764777.636551:0:2380:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1775764777.636591:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967013376:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764777.638586:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764777.638703:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764777.638725:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764777.638782:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764777.638810:0:11131:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978388096 00000100:00100000:1.0:1775764777.638819:0:11131:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978388096 00000100:00100000:1.0:1775764777.638862:0:11131:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978388224 00000100:00100000:1.0:1775764777.638865:0:11131:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978388224 00000100:00100000:1.0:1775764777.638875:0:11131:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978388352 00000100:00100000:1.0:1775764777.638878:0:11131:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978388352 00000100:00100000:1.0:1775764777.638886:0:11131:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978388480 00000100:00100000:1.0:1775764777.638888:0:11131:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978388480 00000100:00100000:1.0:1775764777.638900:0:11131:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24189 00000100:00100000:1.0:1775764777.638908:0:11131:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e6680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+7:12181:x1862022978388096:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764777.638928:0:11131:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4034b9e/0x36f9bdc9e421c125 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x660:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f87f7 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764777.638998:0:11131:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764777.639005:0:11131:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e6680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+7:12181:x1862022978388096:12345-192.168.202.146@tcp:104: Request processed in 97us (428us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764777.639012:0:11131:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24189 00000100:00100000:1.0:1775764777.639031:0:11131:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24190 00000100:00100000:1.0:1775764777.639037:0:11131:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e7800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+6:16287:x1862022978388224:12345-192.168.202.146@tcp:104: 00000100:00100000:0.0:1775764777.644655:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24191 00000100:00100000:0.0:1775764777.644691:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e5c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+6:12187:x1862022978388352:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764777.644711:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000b672b57f/0x36f9bdc9e41cadef lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x5e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eeebf expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 58 lru_type: 1 00010000:00010000:0.0:1775764777.644827:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:0.0:1775764777.644834:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e5c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+6:12187:x1862022978388352:12345-192.168.202.146@tcp:104: Request processed in 163us (6107us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764777.644843:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24191 00000100:00100000:0.0:1775764777.644881:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24192 00000100:00100000:0.0:1775764777.644888:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e4380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:12040:x1862022978388480:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764777.644897:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c23797f9/0x36f9bdc9e41caec8 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x61:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef0b7 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 38 lru_type: 1 00010000:00010000:0.0:1775764777.644924:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:0.0:1775764777.644929:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e4380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:12040:x1862022978388480:12345-192.168.202.146@tcp:104: Request processed in 42us (6148us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764777.644935:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24192 00010000:00010000:0.0:1775764777.644959:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:0.0:1775764777.644966:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4034b9e/0x36f9bdc9e421c125 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x660:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f87f7 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:0.0:1775764777.644974:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9941480c00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:0.0:1775764777.644980:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4034b9e/0x36f9bdc9e421c125 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x660:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e5f87f7 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:0.0:1775764777.645067:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4034b9e/0x36f9bdc9e421c125 lrc: 2/0,0 mode: --/PW res: [0x240000403:0x660:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f87f7 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:0.0:1775764777.645075:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764777.645106:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4034b9e/0x36f9bdc9e421c125 lrc: 1/0,0 mode: --/PW res: [0x240000403:0x660:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f87f7 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:0.0:1775764777.645114:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4034b9e/0x36f9bdc9e421c125 lrc: 0/0,0 mode: --/PW res: [0x240000403:0x660:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f87f7 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:0.0:1775764777.645137:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1775764777.645140:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000b672b57f/0x36f9bdc9e41cadef lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x5e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eeebf expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 58 lru_type: 1 00010000:00010000:0.0:1775764777.645148:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9966c09600 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:0.0:1775764777.645151:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000b672b57f/0x36f9bdc9e41cadef lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x5e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4eeebf expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 58 lru_type: 1 00010000:00010000:0.0:1775764777.645185:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000b672b57f/0x36f9bdc9e41cadef lrc: 2/0,0 mode: --/PW res: [0x240000403:0x5e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eeebf expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 58 lru_type: 1 00010000:00010000:0.0:1775764777.645197:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764777.645203:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000b672b57f/0x36f9bdc9e41cadef lrc: 1/0,0 mode: --/PW res: [0x240000403:0x5e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eeebf expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 58 lru_type: 1 00010000:00010000:0.0:1775764777.645209:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000b672b57f/0x36f9bdc9e41cadef lrc: 0/0,0 mode: --/PW res: [0x240000403:0x5e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eeebf expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 58 lru_type: 1 00010000:00010000:0.0:1775764777.645222:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1775764777.645225:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c23797f9/0x36f9bdc9e41caec8 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x61:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef0b7 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 38 lru_type: 1 00010000:00010000:0.0:1775764777.645231:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9966c09200 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:0.0:1775764777.645234:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c23797f9/0x36f9bdc9e41caec8 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x61:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ef0b7 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 38 lru_type: 1 00010000:00010000:0.0:1775764777.645264:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c23797f9/0x36f9bdc9e41caec8 lrc: 2/0,0 mode: --/PW res: [0x240000403:0x61:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef0b7 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 38 lru_type: 1 00010000:00010000:0.0:1775764777.645270:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764777.645275:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c23797f9/0x36f9bdc9e41caec8 lrc: 1/0,0 mode: --/PW res: [0x240000403:0x61:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef0b7 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 38 lru_type: 1 00010000:00010000:0.0:1775764777.645282:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c23797f9/0x36f9bdc9e41caec8 lrc: 0/0,0 mode: --/PW res: [0x240000403:0x61:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef0b7 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 38 lru_type: 1 00010000:00010000:0.0:1775764777.645293:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764777.645295:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764777.645297:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764777.645304:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764777.645603:0:11131:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a81a0d9c/0x36f9bdc9e41caf07 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x62:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef15f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 41 lru_type: 1 00000100:00100000:1.0:1775764777.645627:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967013504:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764777.645695:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967013760:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764777.645722:0:2380:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1775764777.645728:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967013632:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1775764777.645882:0:11131:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764777.645888:0:11131:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e7800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:16287:x1862022978388224:12345-192.168.202.146@tcp:104: Request processed in 6851us (7185us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764777.645895:0:11131:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24190 00010000:00010000:1.0:1775764777.645911:0:11138:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764777.645916:0:11138:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a81a0d9c/0x36f9bdc9e41caf07 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x62:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef15f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 41 lru_type: 1 00010000:00010000:1.0:1775764777.645924:0:11138:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9966c08c00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:1.0:1775764777.645928:0:11138:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a81a0d9c/0x36f9bdc9e41caf07 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x62:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ef15f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 41 lru_type: 1 00010000:00010000:1.0:1775764777.645986:0:11138:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a81a0d9c/0x36f9bdc9e41caf07 lrc: 2/0,0 mode: --/PW res: [0x240000403:0x62:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef15f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 41 lru_type: 1 00010000:00010000:1.0:1775764777.645993:0:11138:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764777.646005:0:11138:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a81a0d9c/0x36f9bdc9e41caf07 lrc: 1/0,0 mode: --/PW res: [0x240000403:0x62:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef15f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 41 lru_type: 1 00010000:00010000:1.0:1775764777.646011:0:11138:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a81a0d9c/0x36f9bdc9e41caf07 lrc: 0/0,0 mode: --/PW res: [0x240000403:0x62:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef15f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 41 lru_type: 1 00010000:00010000:1.0:1775764777.646029:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764777.646031:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764777.646032:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764777.646041:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967013888:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1775764777.647144:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764777.694404:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764777.694454:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764777.694478:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764777.694502:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764777.694644:0:5561:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1775764777.694663:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978388608 00000100:00100000:1.0:1775764777.694672:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978388608 00000100:00100000:1.0:1775764777.694713:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978388736 00000100:00100000:1.0:1775764777.694716:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978388736 00000100:00100000:1.0:1775764777.694729:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978388992 00000100:00100000:1.0:1775764777.694731:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978388992 00000100:00100000:1.0:1775764777.694765:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24193 00000100:00100000:1.0:1775764777.694773:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e6d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+6:12215:x1862022978388608:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764777.694792:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ecd712f3/0x36f9bdc9e421c221 lrc: 2/0,0 mode: PR/PR res: [0x240000403:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5f8a3c expref: -99 pid: 19852 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764777.694854:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764777.694860:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e6d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+6:12215:x1862022978388608:12345-192.168.202.146@tcp:104: Request processed in 88us (465us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764777.694866:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24193 00000100:00100000:1.0:1775764777.694882:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24194 00000100:00100000:1.0:1775764777.694887:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e4a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:12190:x1862022978388736:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764777.694896:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000e953e3b/0x36f9bdc9e41cb0ce lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x68:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef5cd expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 0 00010000:00010000:1.0:1775764777.694933:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1775764777.694937:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e4a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:12190:x1862022978388736:12345-192.168.202.146@tcp:104: Request processed in 51us (484us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764777.694942:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24194 00000100:00100000:1.0:1775764777.694954:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24195 00000100:00100000:1.0:1775764777.694959:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9966bf7450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:12188:x1862022978388992:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764777.694966:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000054126077/0x36f9bdc9e41cae6d lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x60:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eefec expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 39 lru_type: 1 00010000:00010000:1.0:1775764777.694990:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:1.0:1775764777.694994:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9966bf7450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:12188:x1862022978388992:12345-192.168.202.146@tcp:104: Request processed in 36us (517us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764777.695001:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24195 00010000:00010000:1.0:1775764777.695156:0:11210:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:1.0:1775764777.695162:0:11210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ecd712f3/0x36f9bdc9e421c221 lrc: 2/0,0 mode: PR/PR res: [0x240000403:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5f8a3c expref: -99 pid: 19852 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764777.703032:0:11210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99668b9400 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:1.0:1775764777.703123:0:11210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ecd712f3/0x36f9bdc9e421c221 lrc: 3/0,0 mode: PR/PR res: [0x240000403:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x428400010000 nid: local remote: 0x2aac6a50e5f8a3c expref: -99 pid: 19852 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764777.704162:0:11210:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ecd712f3/0x36f9bdc9e421c221 lrc: 2/0,0 mode: --/PR res: [0x240000403:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5f8a3c expref: -99 pid: 19852 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764777.704263:0:11210:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764777.704281:0:11210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ecd712f3/0x36f9bdc9e421c221 lrc: 1/0,0 mode: --/PR res: [0x240000403:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5f8a3c expref: -99 pid: 19852 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764777.704288:0:11210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ecd712f3/0x36f9bdc9e421c221 lrc: 0/0,0 mode: --/PR res: [0x240000403:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5f8a3c expref: -99 pid: 19852 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764777.704314:0:11210:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1775764777.704320:0:11210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000e953e3b/0x36f9bdc9e41cb0ce lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x68:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef5cd expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 0 00010000:00010000:1.0:1775764777.704327:0:11210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99435d4a00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:1.0:1775764777.704331:0:11210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000e953e3b/0x36f9bdc9e41cb0ce lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x68:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ef5cd expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 0 00010000:00010000:1.0:1775764777.704370:0:11210:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000e953e3b/0x36f9bdc9e41cb0ce lrc: 2/0,0 mode: --/PW res: [0x240000403:0x68:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef5cd expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 0 00010000:00010000:1.0:1775764777.704377:0:11210:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764777.704383:0:11210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000e953e3b/0x36f9bdc9e41cb0ce lrc: 1/0,0 mode: --/PW res: [0x240000403:0x68:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef5cd expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 0 00010000:00010000:1.0:1775764777.704389:0:11210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000e953e3b/0x36f9bdc9e41cb0ce lrc: 0/0,0 mode: --/PW res: [0x240000403:0x68:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef5cd expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 0 00010000:00010000:1.0:1775764777.704402:0:11210:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764777.704406:0:11210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000054126077/0x36f9bdc9e41cae6d lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x60:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eefec expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 39 lru_type: 1 00010000:00010000:1.0:1775764777.705541:0:11210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f994ed0de00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:1.0:1775764777.705548:0:11210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000054126077/0x36f9bdc9e41cae6d lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x60:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4eefec expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 39 lru_type: 1 00010000:00010000:1.0:1775764777.705734:0:11210:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000054126077/0x36f9bdc9e41cae6d lrc: 2/0,0 mode: --/PW res: [0x240000403:0x60:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eefec expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 39 lru_type: 1 00010000:00010000:1.0:1775764777.705768:0:11210:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764777.705778:0:11210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000054126077/0x36f9bdc9e41cae6d lrc: 1/0,0 mode: --/PW res: [0x240000403:0x60:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eefec expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 39 lru_type: 1 00010000:00010000:1.0:1775764777.705785:0:11210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000054126077/0x36f9bdc9e41cae6d lrc: 0/0,0 mode: --/PW res: [0x240000403:0x60:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eefec expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 39 lru_type: 1 00010000:00010000:1.0:1775764777.705808:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764777.705810:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764777.705813:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764777.705823:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764777.705829:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764777.705853:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967014144:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764777.705971:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967012480:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764777.705982:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967012736:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764777.705991:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0cbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967012608:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764777.706016:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1775764777.706021:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967014016:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764777.706058:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967014272:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764777.706098:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967012864:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764777.706108:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967012992:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764777.707674:0:5563:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978388864 00000100:00100000:3.0:1775764777.707686:0:5563:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978388864 00000100:00100000:3.0:1775764777.707773:0:5563:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 1126 00000100:00100000:3.0:1775764777.707783:0:5563:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e4000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:12193:x1862022978388864:12345-192.168.202.146@tcp:104: 00010000:00010000:3.0:1775764777.707799:0:5563:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000008eb5a6c2/0x36f9bdc9e41caf4d lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x63:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef207 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 58 lru_type: 1 00010000:00010000:3.0:1775764777.707863:0:5563:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1775764777.707868:0:5563:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e4000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:12193:x1862022978388864:12345-192.168.202.146@tcp:104: Request processed in 89us (13366us total) trans 0 rc 0/0 00000100:00100000:3.0:1775764777.707875:0:5563:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 1126 00000100:00100000:3.0:1775764777.707926:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967013248:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764777.707936:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967013120:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1775764777.709841:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1775764777.709854:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000008eb5a6c2/0x36f9bdc9e41caf4d lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x63:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef207 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 58 lru_type: 1 00010000:00010000:0.0:1775764777.709867:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f994ed0c400 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:0.0:1775764777.709873:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000008eb5a6c2/0x36f9bdc9e41caf4d lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x63:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ef207 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 58 lru_type: 1 00010000:00010000:0.0:1775764777.709972:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000008eb5a6c2/0x36f9bdc9e41caf4d lrc: 2/0,0 mode: --/PW res: [0x240000403:0x63:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef207 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 58 lru_type: 1 00010000:00010000:0.0:1775764777.709979:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764777.710002:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000008eb5a6c2/0x36f9bdc9e41caf4d lrc: 1/0,0 mode: --/PW res: [0x240000403:0x63:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef207 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 58 lru_type: 1 00010000:00010000:0.0:1775764777.710010:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000008eb5a6c2/0x36f9bdc9e41caf4d lrc: 0/0,0 mode: --/PW res: [0x240000403:0x63:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef207 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 58 lru_type: 1 00010000:00010000:0.0:1775764777.710028:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764777.710030:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764777.710032:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764777.710186:0:2380:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1775764777.710205:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967014400:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764777.715768:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764777.715824:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0ca680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967013632:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764777.715835:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967013504:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764777.715843:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967013760:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764777.715853:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967013376:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764777.715893:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967013888:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764777.715916:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978389248 00000100:00100000:1.0:1775764777.715923:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978389248 00000100:00100000:1.0:1775764777.715971:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24196 00000100:00100000:1.0:1775764777.715981:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e7800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:12181:x1862022978389248:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764777.716003:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001c6eb232/0x36f9bdc9e41cae27 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x5f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eef44 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 58 lru_type: 1 00010000:00010000:1.0:1775764777.716094:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764777.716101:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e7800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:12181:x1862022978389248:12345-192.168.202.146@tcp:104: Request processed in 121us (369us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764777.716108:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24196 00010000:00010000:3.0:1775764777.716505:0:11138:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1775764777.716512:0:11138:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001c6eb232/0x36f9bdc9e41cae27 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x5f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eef44 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 58 lru_type: 1 00010000:00010000:3.0:1775764777.716520:0:11138:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f994f3b9000 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:3.0:1775764777.716526:0:11138:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001c6eb232/0x36f9bdc9e41cae27 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x5f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4eef44 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 58 lru_type: 1 00010000:00010000:3.0:1775764777.716624:0:11138:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001c6eb232/0x36f9bdc9e41cae27 lrc: 2/0,0 mode: --/PW res: [0x240000403:0x5f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eef44 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 58 lru_type: 1 00010000:00010000:3.0:1775764777.716632:0:11138:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764777.716654:0:11138:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001c6eb232/0x36f9bdc9e41cae27 lrc: 1/0,0 mode: --/PW res: [0x240000403:0x5f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eef44 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 58 lru_type: 1 00010000:00010000:3.0:1775764777.716662:0:11138:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001c6eb232/0x36f9bdc9e41cae27 lrc: 0/0,0 mode: --/PW res: [0x240000403:0x5f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eef44 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 58 lru_type: 1 00010000:00010000:3.0:1775764777.716690:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764777.716692:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764777.716693:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764777.716775:0:2382:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1775764777.716789:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967014528:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764777.727373:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764777.727409:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764777.727433:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764777.727452:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764777.727468:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764777.727490:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764777.727520:0:5562:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1775764777.727538:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978390016 00000100:00100000:1.0:1775764777.727549:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978390016 00000100:00100000:1.0:1775764777.727619:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978390144 00000100:00100000:1.0:1775764777.727622:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978390144 00000100:00100000:1.0:1775764777.727632:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978390272 00000100:00100000:1.0:1775764777.727635:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978390272 00000100:00100000:1.0:1775764777.727645:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978390400 00000100:00100000:1.0:1775764777.727648:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978390400 00000100:00100000:1.0:1775764777.727662:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978390528 00000100:00100000:1.0:1775764777.727664:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978390528 00000100:00100000:1.0:1775764777.727673:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978390784 00000100:00100000:1.0:1775764777.727675:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978390784 00000100:00100000:1.0:1775764777.727687:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24197 00000100:00100000:1.0:1775764777.727697:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e5180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+9:16465:x1862022978390016:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764777.727715:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000000b928e73/0x36f9bdc9e41caf70 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x63:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef24d expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 66 lru_type: 1 00010000:00010000:1.0:1775764777.727828:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764777.727835:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e5180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+9:16465:x1862022978390016:12345-192.168.202.146@tcp:104: Request processed in 140us (476us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764777.727842:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24197 00000100:00100000:1.0:1775764777.727860:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24198 00000100:00100000:1.0:1775764777.727865:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e6d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+8:12182:x1862022978390144:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764777.727874:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000022d8547f/0x36f9bdc9e41caf1c lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x62:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef190 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 76 lru_type: 1 00010000:00010000:1.0:1775764777.727900:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1775764777.727905:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e6d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+8:12182:x1862022978390144:12345-192.168.202.146@tcp:104: Request processed in 41us (497us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764777.727912:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24198 00000100:00100000:1.0:1775764777.727924:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24199 00000100:00100000:1.0:1775764777.727930:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e4a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+7:6578:x1862022978390272:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764777.727939:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000b908cb64/0x36f9bdc9e41cafa8 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x64:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef303 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 66 lru_type: 1 00010000:00010000:1.0:1775764777.727966:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:1.0:1775764777.727971:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e4a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+7:6578:x1862022978390272:12345-192.168.202.146@tcp:104: Request processed in 42us (538us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764777.727977:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24199 00000100:00100000:1.0:1775764777.727988:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24200 00000100:00100000:1.0:1775764777.727994:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e4700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+6:16287:x1862022978390400:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764777.728003:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000df037f52/0x36f9bdc9e41cae58 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x5f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eefbb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 69 lru_type: 1 00010000:00010000:1.0:1775764777.728031:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/4 locks to priority blp list, 4 blwis in pool 00000100:00100000:1.0:1775764777.728037:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e4700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+6:16287:x1862022978390400:12345-192.168.202.146@tcp:104: Request processed in 42us (584us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764777.728042:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24200 00000100:00100000:1.0:1775764777.728054:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24201 00000100:00100000:1.0:1775764777.728060:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e5880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:6579:x1862022978390528:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764777.728068:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000260038e6/0x36f9bdc9e41caed6 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x61:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef0e8 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 102 lru_type: 1 00010000:00010000:1.0:1775764777.728091:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/5 locks to priority blp list, 5 blwis in pool 00000100:00100000:1.0:1775764777.728095:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e5880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:6579:x1862022978390528:12345-192.168.202.146@tcp:104: Request processed in 36us (627us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764777.728100:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24201 00000100:00100000:1.0:1775764777.728111:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24202 00000100:00100000:1.0:1775764777.728116:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970e08850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:12187:x1862022978390784:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764777.728123:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c8469d80/0x36f9bdc9e41cae0b lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x5e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eef05 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 39 lru_type: 1 00010000:00010000:1.0:1775764777.728152:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/6 locks to priority blp list, 6 blwis in pool 00000100:00100000:1.0:1775764777.728157:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970e08850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:12187:x1862022978390784:12345-192.168.202.146@tcp:104: Request processed in 41us (667us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764777.728163:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24202 00010000:00010000:0.0:1775764777.728314:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 5 total in blp. (5 blwis in pool) 00010000:00010000:0.0:1775764777.728322:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000000b928e73/0x36f9bdc9e41caf70 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x63:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef24d expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 66 lru_type: 1 00010000:00010000:0.0:1775764777.728336:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f994ed0d000 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:0.0:1775764777.728343:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000000b928e73/0x36f9bdc9e41caf70 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x63:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ef24d expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 66 lru_type: 1 00010000:00010000:0.0:1775764777.728436:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000000b928e73/0x36f9bdc9e41caf70 lrc: 2/0,0 mode: --/PW res: [0x280000403:0x63:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef24d expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 66 lru_type: 1 00010000:00010000:0.0:1775764777.728444:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764777.728470:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000000b928e73/0x36f9bdc9e41caf70 lrc: 1/0,0 mode: --/PW res: [0x280000403:0x63:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef24d expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 66 lru_type: 1 00010000:00010000:0.0:1775764777.728478:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000000b928e73/0x36f9bdc9e41caf70 lrc: 0/0,0 mode: --/PW res: [0x280000403:0x63:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef24d expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 66 lru_type: 1 00010000:00010000:0.0:1775764777.728497:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 4 total in blp. (4 blwis in pool) 00010000:00010000:0.0:1775764777.728501:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000022d8547f/0x36f9bdc9e41caf1c lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x62:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef190 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 76 lru_type: 1 00010000:00010000:0.0:1775764777.728507:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99413d3200 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:0.0:1775764777.728510:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000022d8547f/0x36f9bdc9e41caf1c lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x62:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ef190 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 76 lru_type: 1 00010000:00010000:0.0:1775764777.728546:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000022d8547f/0x36f9bdc9e41caf1c lrc: 2/0,0 mode: --/PW res: [0x280000403:0x62:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef190 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 76 lru_type: 1 00010000:00010000:0.0:1775764777.728569:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764777.728576:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000022d8547f/0x36f9bdc9e41caf1c lrc: 1/0,0 mode: --/PW res: [0x280000403:0x62:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef190 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 76 lru_type: 1 00010000:00010000:0.0:1775764777.728583:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000022d8547f/0x36f9bdc9e41caf1c lrc: 0/0,0 mode: --/PW res: [0x280000403:0x62:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef190 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 76 lru_type: 1 00010000:00010000:0.0:1775764777.728598:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 3 total in blp. (3 blwis in pool) 00010000:00010000:0.0:1775764777.728602:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000b908cb64/0x36f9bdc9e41cafa8 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x64:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef303 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 66 lru_type: 1 00010000:00010000:0.0:1775764777.728607:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9971071600 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:0.0:1775764777.728611:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000b908cb64/0x36f9bdc9e41cafa8 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x64:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ef303 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 66 lru_type: 1 00010000:00010000:0.0:1775764777.728647:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000b908cb64/0x36f9bdc9e41cafa8 lrc: 2/0,0 mode: --/PW res: [0x280000403:0x64:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef303 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 66 lru_type: 1 00010000:00010000:0.0:1775764777.728653:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764777.728658:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000b908cb64/0x36f9bdc9e41cafa8 lrc: 1/0,0 mode: --/PW res: [0x280000403:0x64:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef303 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 66 lru_type: 1 00010000:00010000:0.0:1775764777.728664:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000b908cb64/0x36f9bdc9e41cafa8 lrc: 0/0,0 mode: --/PW res: [0x280000403:0x64:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef303 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 66 lru_type: 1 00010000:00010000:0.0:1775764777.728677:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:0.0:1775764777.728680:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000df037f52/0x36f9bdc9e41cae58 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x5f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eefbb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 69 lru_type: 1 00010000:00010000:0.0:1775764777.728685:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9966c08600 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:0.0:1775764777.728688:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000df037f52/0x36f9bdc9e41cae58 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x5f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4eefbb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 69 lru_type: 1 00010000:00010000:0.0:1775764777.728717:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000df037f52/0x36f9bdc9e41cae58 lrc: 2/0,0 mode: --/PW res: [0x280000403:0x5f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eefbb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 69 lru_type: 1 00010000:00010000:0.0:1775764777.728723:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764777.728727:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000df037f52/0x36f9bdc9e41cae58 lrc: 1/0,0 mode: --/PW res: [0x280000403:0x5f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eefbb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 69 lru_type: 1 00010000:00010000:0.0:1775764777.728733:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000df037f52/0x36f9bdc9e41cae58 lrc: 0/0,0 mode: --/PW res: [0x280000403:0x5f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eefbb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 69 lru_type: 1 00010000:00010000:0.0:1775764777.728772:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1775764777.728775:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000260038e6/0x36f9bdc9e41caed6 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x61:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef0e8 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 102 lru_type: 1 00010000:00010000:0.0:1775764777.728782:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9971204400 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:0.0:1775764777.728785:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000260038e6/0x36f9bdc9e41caed6 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x61:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ef0e8 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 102 lru_type: 1 00010000:00010000:0.0:1775764777.728813:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000260038e6/0x36f9bdc9e41caed6 lrc: 2/0,0 mode: --/PW res: [0x280000403:0x61:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef0e8 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 102 lru_type: 1 00010000:00010000:0.0:1775764777.728824:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764777.728829:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000260038e6/0x36f9bdc9e41caed6 lrc: 1/0,0 mode: --/PW res: [0x280000403:0x61:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef0e8 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 102 lru_type: 1 00010000:00010000:0.0:1775764777.728834:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000260038e6/0x36f9bdc9e41caed6 lrc: 0/0,0 mode: --/PW res: [0x280000403:0x61:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef0e8 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 102 lru_type: 1 00010000:00010000:0.0:1775764777.728848:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1775764777.728851:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c8469d80/0x36f9bdc9e41cae0b lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x5e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eef05 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 39 lru_type: 1 00010000:00010000:0.0:1775764777.728856:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f994ed0d600 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:0.0:1775764777.728859:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c8469d80/0x36f9bdc9e41cae0b lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x5e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4eef05 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 39 lru_type: 1 00010000:00010000:0.0:1775764777.728885:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c8469d80/0x36f9bdc9e41cae0b lrc: 2/0,0 mode: --/PW res: [0x280000403:0x5e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eef05 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 39 lru_type: 1 00010000:00010000:0.0:1775764777.728891:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764777.728896:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c8469d80/0x36f9bdc9e41cae0b lrc: 1/0,0 mode: --/PW res: [0x280000403:0x5e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eef05 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 39 lru_type: 1 00010000:00010000:0.0:1775764777.728901:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000c8469d80/0x36f9bdc9e41cae0b lrc: 0/0,0 mode: --/PW res: [0x280000403:0x5e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eef05 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 39 lru_type: 1 00010000:00010000:0.0:1775764777.728913:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764777.728914:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764777.728916:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764777.728935:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967014656:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764777.728994:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967014912:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764777.729013:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967015168:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764777.729031:0:2380:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:0.0:1775764777.729036:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967014784:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764777.729049:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967015040:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764777.729060:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967015296:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1775764777.729146:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764777.729971:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764777.729978:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764777.729984:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764777.731030:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764777.741937:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764777.741961:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764777.741981:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764777.741998:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764777.742018:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764777.742054:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978391424 00000100:00100000:1.0:1775764777.742064:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978391424 00000100:00100000:3.0:1775764777.742083:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967014528:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764777.742106:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978391680 00000100:00100000:1.0:1775764777.742110:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978391680 00000100:00100000:1.0:1775764777.742120:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978391808 00000100:00100000:1.0:1775764777.742121:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978391808 00000100:00100000:1.0:1775764777.742128:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978391936 00000100:00100000:1.0:1775764777.742130:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978391936 00000100:00100000:1.0:1775764777.742136:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978392448 00000100:00100000:1.0:1775764777.742138:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978392448 00000100:00100000:1.0:1775764777.742150:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24203 00000100:00100000:1.0:1775764777.742158:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e7800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+6:12186:x1862022978391424:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764777.742186:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000007c16d2ef/0x36f9bdc9e41cad9b lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x5d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eedf4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 48 lru_type: 1 00010000:00010000:1.0:1775764777.742263:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764777.742270:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e7800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+6:12186:x1862022978391424:12345-192.168.202.146@tcp:104: Request processed in 113us (345us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764777.742277:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24203 00000100:00100000:1.0:1775764777.742295:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24204 00000100:00100000:1.0:1775764777.742300:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e5180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:12180:x1862022978391680:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764777.742307:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000050510d50/0x36f9bdc9e41cad55 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x5c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eed4c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 38 lru_type: 1 00010000:00010000:1.0:1775764777.742343:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1775764777.742348:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e5180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:12180:x1862022978391680:12345-192.168.202.146@tcp:104: Request processed in 48us (388us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764777.742353:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24204 00000100:00100000:1.0:1775764777.742363:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24205 00000100:00100000:1.0:1775764777.742367:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e6d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:12040:x1862022978391808:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764777.742375:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f970e87b/0x36f9bdc9e41cadcc lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x5d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eee6b expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 39 lru_type: 1 00010000:00010000:1.0:1775764777.742404:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:1.0:1775764777.742409:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e6d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:12040:x1862022978391808:12345-192.168.202.146@tcp:104: Request processed in 41us (428us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764777.742414:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24205 00000100:00100000:1.0:1775764777.742424:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24206 00000100:00100000:1.0:1775764777.742430:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e4a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:12188:x1862022978391936:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764777.742439:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006a530be3/0x36f9bdc9e420da48 lrc: 2/0,0 mode: PR/PR res: [0x280000403:0x5c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5c9984 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 31 lru_type: 1 00010000:00010000:1.0:1775764777.742471:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/4 locks to priority blp list, 4 blwis in pool 00000100:00100000:1.0:1775764777.742476:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e4a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:12188:x1862022978391936:12345-192.168.202.146@tcp:104: Request processed in 47us (478us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764777.742482:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24206 00000100:00100000:1.0:1775764777.742493:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24207 00000100:00100000:1.0:1775764777.742498:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e4700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:12193:x1862022978392448:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764777.742507:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000070c338a1/0x36f9bdc9e41cb09d lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x67:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef556 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 29 lru_type: 1 00010000:00010000:1.0:1775764777.742534:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/5 locks to priority blp list, 5 blwis in pool 00000100:00100000:1.0:1775764777.742545:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e4700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:12193:x1862022978392448:12345-192.168.202.146@tcp:104: Request processed in 41us (520us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764777.742550:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24207 00000100:00100000:0.0:1775764777.742797:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967014016:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764777.742808:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967014272:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764777.742816:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967014144:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764777.742848:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967014400:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1775764777.743538:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 4 total in blp. (4 blwis in pool) 00010000:00010000:3.0:1775764777.743547:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000007c16d2ef/0x36f9bdc9e41cad9b lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x5d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eedf4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 48 lru_type: 1 00010000:00010000:3.0:1775764777.743569:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9971204c00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:3.0:1775764777.743575:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000007c16d2ef/0x36f9bdc9e41cad9b lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x5d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4eedf4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 48 lru_type: 1 00010000:00010000:3.0:1775764777.743661:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000007c16d2ef/0x36f9bdc9e41cad9b lrc: 2/0,0 mode: --/PW res: [0x240000403:0x5d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eedf4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 48 lru_type: 1 00010000:00010000:3.0:1775764777.743668:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764777.743690:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000007c16d2ef/0x36f9bdc9e41cad9b lrc: 1/0,0 mode: --/PW res: [0x240000403:0x5d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eedf4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 48 lru_type: 1 00010000:00010000:3.0:1775764777.743697:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000007c16d2ef/0x36f9bdc9e41cad9b lrc: 0/0,0 mode: --/PW res: [0x240000403:0x5d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eedf4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 48 lru_type: 1 00010000:00010000:3.0:1775764777.743719:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 3 total in blp. (3 blwis in pool) 00010000:00010000:3.0:1775764777.743724:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000050510d50/0x36f9bdc9e41cad55 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x5c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eed4c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 38 lru_type: 1 00010000:00010000:3.0:1775764777.743730:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f994f3d8a00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:3.0:1775764777.743733:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000050510d50/0x36f9bdc9e41cad55 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x5c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4eed4c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 38 lru_type: 1 00010000:00010000:3.0:1775764777.743787:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000050510d50/0x36f9bdc9e41cad55 lrc: 2/0,0 mode: --/PW res: [0x240000403:0x5c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eed4c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 38 lru_type: 1 00010000:00010000:3.0:1775764777.743793:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764777.743799:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000050510d50/0x36f9bdc9e41cad55 lrc: 1/0,0 mode: --/PW res: [0x240000403:0x5c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eed4c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 38 lru_type: 1 00010000:00010000:3.0:1775764777.743806:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000050510d50/0x36f9bdc9e41cad55 lrc: 0/0,0 mode: --/PW res: [0x240000403:0x5c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eed4c expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 38 lru_type: 1 00010000:00010000:3.0:1775764777.743819:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:3.0:1775764777.743822:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f970e87b/0x36f9bdc9e41cadcc lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x5d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eee6b expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 39 lru_type: 1 00010000:00010000:3.0:1775764777.743828:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f994ed0c800 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:3.0:1775764777.743831:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f970e87b/0x36f9bdc9e41cadcc lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x5d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4eee6b expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 39 lru_type: 1 00010000:00010000:3.0:1775764777.743861:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f970e87b/0x36f9bdc9e41cadcc lrc: 2/0,0 mode: --/PW res: [0x280000403:0x5d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eee6b expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 39 lru_type: 1 00010000:00010000:3.0:1775764777.743873:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764777.743877:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f970e87b/0x36f9bdc9e41cadcc lrc: 1/0,0 mode: --/PW res: [0x280000403:0x5d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eee6b expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 39 lru_type: 1 00010000:00010000:3.0:1775764777.743883:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000f970e87b/0x36f9bdc9e41cadcc lrc: 0/0,0 mode: --/PW res: [0x280000403:0x5d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eee6b expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 39 lru_type: 1 00010000:00010000:3.0:1775764777.743895:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1775764777.743899:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006a530be3/0x36f9bdc9e420da48 lrc: 2/0,0 mode: PR/PR res: [0x280000403:0x5c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5c9984 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 31 lru_type: 1 00010000:00010000:3.0:1775764777.743904:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99710c9400 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:3.0:1775764777.743907:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006a530be3/0x36f9bdc9e420da48 lrc: 3/0,0 mode: PR/PR res: [0x280000403:0x5c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x428400010000 nid: local remote: 0x2aac6a50e5c9984 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 31 lru_type: 1 00010000:00010000:3.0:1775764777.743930:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006a530be3/0x36f9bdc9e420da48 lrc: 2/0,0 mode: --/PR res: [0x280000403:0x5c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5c9984 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 31 lru_type: 1 00010000:00010000:3.0:1775764777.743936:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764777.743940:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006a530be3/0x36f9bdc9e420da48 lrc: 1/0,0 mode: --/PR res: [0x280000403:0x5c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5c9984 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 31 lru_type: 1 00010000:00010000:3.0:1775764777.743946:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000006a530be3/0x36f9bdc9e420da48 lrc: 0/0,0 mode: --/PR res: [0x280000403:0x5c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5c9984 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 31 lru_type: 1 00010000:00010000:3.0:1775764777.743956:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1775764777.743960:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000070c338a1/0x36f9bdc9e41cb09d lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x67:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef556 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 29 lru_type: 1 00010000:00010000:3.0:1775764777.743965:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99413d2a00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:3.0:1775764777.743968:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000070c338a1/0x36f9bdc9e41cb09d lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x67:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ef556 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 29 lru_type: 1 00010000:00010000:3.0:1775764777.743990:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000070c338a1/0x36f9bdc9e41cb09d lrc: 2/0,0 mode: --/PW res: [0x240000403:0x67:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef556 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 29 lru_type: 1 00010000:00010000:3.0:1775764777.743996:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764777.743999:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000070c338a1/0x36f9bdc9e41cb09d lrc: 1/0,0 mode: --/PW res: [0x240000403:0x67:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef556 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 29 lru_type: 1 00010000:00010000:3.0:1775764777.744005:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000070c338a1/0x36f9bdc9e41cb09d lrc: 0/0,0 mode: --/PW res: [0x240000403:0x67:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef556 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 29 lru_type: 1 00010000:00010000:3.0:1775764777.744015:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764777.744017:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764777.744019:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764777.744035:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967015552:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764777.744118:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967015808:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764777.744138:0:2381:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [1->0] 00000100:00100000:3.0:1775764777.744143:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967015424:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764777.744158:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967015680:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764777.744183:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967015936:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764777.744841:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967014784:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764777.744852:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967015040:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764777.744859:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0cb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967015296:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764777.744866:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967014656:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764777.744874:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967014912:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764777.744881:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967015168:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1775764777.747427:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764777.747439:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764777.747544:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764777.747551:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764777.769376:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764777.769429:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764777.769459:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764777.769491:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764777.769511:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764777.769531:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764777.769574:0:5561:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1775764777.769593:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978392576 00000100:00100000:0.0:1775764777.769602:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978392576 00000100:00100000:0.0:1775764777.769647:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978392704 00000100:00100000:0.0:1775764777.769650:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978392704 00000100:00100000:0.0:1775764777.769668:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24208 00000100:00100000:0.0:1775764777.769676:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0cad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:8216:x1862022978392576:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764777.769695:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000002c55d5c6/0x36f9bdc9e41cad0f lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x5b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eeca4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 38 lru_type: 1 00010000:00010000:0.0:1775764777.769781:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764777.769787:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0cad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:8216:x1862022978392576:12345-192.168.202.146@tcp:104: Request processed in 112us (423us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764777.769795:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24208 00000100:00100000:0.0:1775764777.769810:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24209 00000100:00100000:0.0:1775764777.769816:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970e0ac50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:16465:x1862022978392704:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764777.769824:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001fc9e7d4/0x36f9bdc9e41cacc9 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x5a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eebfc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 48 lru_type: 1 00010000:00010000:0.0:1775764777.769857:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:0.0:1775764777.769863:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970e0ac50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:16465:x1862022978392704:12345-192.168.202.146@tcp:104: Request processed in 47us (434us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764777.769869:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24209 00010000:00010000:0.0:1775764777.769902:0:11138:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1775764777.769909:0:11138:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000002c55d5c6/0x36f9bdc9e41cad0f lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x5b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eeca4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 38 lru_type: 1 00010000:00010000:0.0:1775764777.769918:0:11138:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9971204a00 already unused, calling callback (ffffffffc12102c0) 00000100:00100000:2.0:1775764777.769920:0:5564:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978392832 00010000:00010000:0.0:1775764777.769924:0:11138:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000002c55d5c6/0x36f9bdc9e41cad0f lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x5b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4eeca4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 38 lru_type: 1 00000100:00100000:2.0:1775764777.769926:0:5564:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978392832 00000100:00100000:2.0:1775764777.769950:0:5564:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978393088 00000100:00100000:2.0:1775764777.769954:0:5564:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978393088 00000100:00100000:2.0:1775764777.769969:0:5564:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 1127 00000100:00100000:2.0:1775764777.769978:0:5564:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c8380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+5:16287:x1862022978392832:12345-192.168.202.146@tcp:104: 00010000:00010000:2.0:1775764777.769993:0:5564:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ac098a84/0x36f9bdc9e41cac83 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x59:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eeb54 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 21 lru_type: 0 00010000:00010000:0.0:1775764777.770018:0:11138:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000002c55d5c6/0x36f9bdc9e41cad0f lrc: 2/0,0 mode: --/PW res: [0x240000403:0x5b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eeca4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 38 lru_type: 1 00010000:00010000:0.0:1775764777.770026:0:11138:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764777.770034:0:5564:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:2.0:1775764777.770041:0:5564:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c8380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+5:16287:x1862022978392832:12345-192.168.202.146@tcp:104: Request processed in 64us (580us total) trans 0 rc 0/0 00000100:00100000:2.0:1775764777.770048:0:5564:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 1127 00010000:00010000:0.0:1775764777.770052:0:11138:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000002c55d5c6/0x36f9bdc9e41cad0f lrc: 1/0,0 mode: --/PW res: [0x240000403:0x5b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eeca4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 38 lru_type: 1 00000100:00100000:2.0:1775764777.770061:0:5564:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978393344 00010000:00010000:0.0:1775764777.770062:0:11138:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000002c55d5c6/0x36f9bdc9e41cad0f lrc: 0/0,0 mode: --/PW res: [0x240000403:0x5b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eeca4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 38 lru_type: 1 00000100:00100000:2.0:1775764777.770065:0:5564:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978393344 00010000:00010000:0.0:1775764777.770088:0:11138:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00000100:00100000:2.0:1775764777.770091:0:5564:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978393600 00010000:00010000:0.0:1775764777.770092:0:11138:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001fc9e7d4/0x36f9bdc9e41cacc9 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x5a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eebfc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 48 lru_type: 1 00000100:00100000:2.0:1775764777.770096:0:5564:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978393600 00010000:00010000:0.0:1775764777.770109:0:11138:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970adfc00 already unused, calling callback (ffffffffc12102c0) 00000100:00100000:2.0:1775764777.770114:0:5564:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 1128 00010000:00010000:0.0:1775764777.770115:0:11138:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001fc9e7d4/0x36f9bdc9e41cacc9 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x5a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4eebfc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 48 lru_type: 1 00000100:00100000:2.0:1775764777.770123:0:5564:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c8000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:12184:x1862022978393088:12345-192.168.202.146@tcp:104: 00010000:00010000:2.0:1775764777.770134:0:5564:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000073099ecc/0x36f9bdc9e41cac4b lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x58:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eeacf expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 30 lru_type: 1 00010000:00010000:0.0:1775764777.770158:0:11138:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001fc9e7d4/0x36f9bdc9e41cacc9 lrc: 2/0,0 mode: --/PW res: [0x240000403:0x5a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eebfc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 48 lru_type: 1 00010000:00010000:2.0:1775764777.770169:0:5564:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:2.0:1775764777.770175:0:5564:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c8000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:12184:x1862022978393088:12345-192.168.202.146@tcp:104: Request processed in 54us (684us total) trans 0 rc 0/0 00000100:00100000:2.0:1775764777.770180:0:5564:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 1128 00000100:00100000:2.0:1775764777.770192:0:5564:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 1129 00000100:00100000:2.0:1775764777.770199:0:5564:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0cb800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+5:12190:x1862022978393344:12345-192.168.202.146@tcp:104: 00010000:00010000:2.0:1775764777.770208:0:5564:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000003d943106/0x36f9bdc9e420da09 lrc: 2/0,0 mode: PR/PR res: [0x280000403:0x5b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5c98dc expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 31 lru_type: 1 00010000:00010000:2.0:1775764777.770241:0:5564:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:2.0:1775764777.770246:0:5564:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0cb800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+5:12190:x1862022978393344:12345-192.168.202.146@tcp:104: Request processed in 49us (735us total) trans 0 rc 0/0 00000100:00100000:2.0:1775764777.770251:0:5564:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 1129 00000100:00100000:2.0:1775764777.770262:0:5564:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 1130 00000100:00100000:2.0:1775764777.770267:0:5564:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c9c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:12041:x1862022978393600:12345-192.168.202.146@tcp:104: 00010000:00010000:2.0:1775764777.770275:0:5564:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000d1791e23/0x36f9bdc9e41cad01 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x5a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eec96 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 163 lru_type: 1 00010000:00010000:2.0:1775764777.770300:0:5564:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/4 locks to priority blp list, 4 blwis in pool 00000100:00100000:2.0:1775764777.770304:0:5564:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c9c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:12041:x1862022978393600:12345-192.168.202.146@tcp:104: Request processed in 38us (773us total) trans 0 rc 0/0 00000100:00100000:2.0:1775764777.770309:0:5564:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 1130 00010000:00010000:2.0:1775764777.770324:0:19800:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 3 total in blp. (3 blwis in pool) 00010000:00010000:2.0:1775764777.770330:0:19800:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ac098a84/0x36f9bdc9e41cac83 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x59:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eeb54 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 21 lru_type: 0 00010000:00010000:2.0:1775764777.770337:0:19800:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970adf400 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:2.0:1775764777.770341:0:19800:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ac098a84/0x36f9bdc9e41cac83 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x59:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4eeb54 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 21 lru_type: 0 00010000:00010000:2.0:1775764777.770388:0:19800:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ac098a84/0x36f9bdc9e41cac83 lrc: 2/0,0 mode: --/PW res: [0x240000403:0x59:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eeb54 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 21 lru_type: 0 00010000:00010000:2.0:1775764777.770395:0:19800:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764777.770414:0:19800:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ac098a84/0x36f9bdc9e41cac83 lrc: 1/0,0 mode: --/PW res: [0x240000403:0x59:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eeb54 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 21 lru_type: 0 00010000:00010000:2.0:1775764777.770421:0:19800:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ac098a84/0x36f9bdc9e41cac83 lrc: 0/0,0 mode: --/PW res: [0x240000403:0x59:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eeb54 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 21 lru_type: 0 00010000:00010000:2.0:1775764777.770438:0:19800:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:2.0:1775764777.770441:0:19800:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000073099ecc/0x36f9bdc9e41cac4b lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x58:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eeacf expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 30 lru_type: 1 00010000:00010000:2.0:1775764777.770448:0:19800:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970ade400 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:2.0:1775764777.770452:0:19800:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000073099ecc/0x36f9bdc9e41cac4b lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x58:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4eeacf expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 30 lru_type: 1 00010000:00010000:2.0:1775764777.770489:0:19800:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000073099ecc/0x36f9bdc9e41cac4b lrc: 2/0,0 mode: --/PW res: [0x240000403:0x58:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eeacf expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 30 lru_type: 1 00010000:00010000:2.0:1775764777.770496:0:19800:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764777.770502:0:19800:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000073099ecc/0x36f9bdc9e41cac4b lrc: 1/0,0 mode: --/PW res: [0x240000403:0x58:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eeacf expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 30 lru_type: 1 00010000:00010000:2.0:1775764777.770509:0:19800:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000073099ecc/0x36f9bdc9e41cac4b lrc: 0/0,0 mode: --/PW res: [0x240000403:0x58:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eeacf expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 30 lru_type: 1 00010000:00010000:2.0:1775764777.770525:0:19800:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1775764777.770529:0:19800:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000003d943106/0x36f9bdc9e420da09 lrc: 2/0,0 mode: PR/PR res: [0x280000403:0x5b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5c98dc expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 31 lru_type: 1 00010000:00010000:1.0:1775764777.770538:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764777.770545:0:19800:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970641600 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:1.0:1775764777.770546:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000d1791e23/0x36f9bdc9e41cad01 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x5a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eec96 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 163 lru_type: 1 00010000:00010000:2.0:1775764777.770549:0:19800:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000003d943106/0x36f9bdc9e420da09 lrc: 3/0,0 mode: PR/PR res: [0x280000403:0x5b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x428400010000 nid: local remote: 0x2aac6a50e5c98dc expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 31 lru_type: 1 00010000:00010000:1.0:1775764777.770574:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9966c08a00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:1.0:1775764777.770580:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000d1791e23/0x36f9bdc9e41cad01 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x5a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4eec96 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 163 lru_type: 1 00010000:00010000:2.0:1775764777.770608:0:19800:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000003d943106/0x36f9bdc9e420da09 lrc: 2/0,0 mode: --/PR res: [0x280000403:0x5b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5c98dc expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 31 lru_type: 1 00010000:00010000:2.0:1775764777.770614:0:19800:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764777.770619:0:19800:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000003d943106/0x36f9bdc9e420da09 lrc: 1/0,0 mode: --/PR res: [0x280000403:0x5b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5c98dc expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 31 lru_type: 1 00010000:00010000:2.0:1775764777.770625:0:19800:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000003d943106/0x36f9bdc9e420da09 lrc: 0/0,0 mode: --/PR res: [0x280000403:0x5b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5c98dc expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 31 lru_type: 1 00010000:00010000:2.0:1775764777.770639:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764777.770641:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764777.770642:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764777.770643:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000d1791e23/0x36f9bdc9e41cad01 lrc: 2/0,0 mode: --/PW res: [0x280000403:0x5a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eec96 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 163 lru_type: 1 00010000:00010000:2.0:1775764777.770649:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764777.770652:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764777.770662:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000d1791e23/0x36f9bdc9e41cad01 lrc: 1/0,0 mode: --/PW res: [0x280000403:0x5a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eec96 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 163 lru_type: 1 00010000:00010000:1.0:1775764777.770669:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000d1791e23/0x36f9bdc9e41cad01 lrc: 0/0,0 mode: --/PW res: [0x280000403:0x5a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eec96 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 163 lru_type: 1 00010000:00010000:1.0:1775764777.770690:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764777.770692:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764777.770694:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764777.770702:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764777.771649:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967016320:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764777.771772:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967016576:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764777.771806:0:2381:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1775764777.771812:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967016448:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1775764777.771889:0:11138:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764777.771904:0:11138:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001fc9e7d4/0x36f9bdc9e41cacc9 lrc: 1/0,0 mode: --/PW res: [0x240000403:0x5a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eebfc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 48 lru_type: 1 00010000:00010000:0.0:1775764777.771916:0:11138:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000001fc9e7d4/0x36f9bdc9e41cacc9 lrc: 0/0,0 mode: --/PW res: [0x240000403:0x5a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eebfc expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 48 lru_type: 1 00010000:00010000:0.0:1775764777.771938:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764777.771940:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764777.771942:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764777.771954:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967016064:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764777.771985:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967016192:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764777.772007:0:2380:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1775764777.772014:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967016704:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1775764777.772045:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764777.808644:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967016320:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764777.808669:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967016576:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764777.808678:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967015424:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764777.808688:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967015680:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764777.808695:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967015936:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764777.808702:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967015552:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764777.808709:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967015808:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764777.808767:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967016448:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764777.808829:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967016064:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764777.809599:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764777.809650:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764777.809681:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764777.809710:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764777.809733:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764777.809770:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764777.809810:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764777.809828:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764777.809858:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764777.809909:0:11131:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:2.0:1775764777.809915:0:5564:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:2.0:1775764777.809929:0:5564:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978395648 00000100:00100000:0.0:1775764777.809930:0:11131:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978394112 00000100:00100000:0.0:1775764777.809937:0:11131:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978394112 00000100:00100000:2.0:1775764777.809938:0:5564:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978395648 00000100:00100000:0.0:1775764777.809980:0:11131:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978394368 00000100:00100000:0.0:1775764777.809984:0:11131:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978394368 00000100:00100000:2.0:1775764777.809985:0:5564:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978396160 00000100:00100000:2.0:1775764777.809990:0:5564:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978396160 00000100:00100000:0.0:1775764777.810001:0:11131:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978395008 00000100:00100000:0.0:1775764777.810005:0:11131:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978395008 00000100:00100000:2.0:1775764777.810015:0:5564:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 1131 00000100:00100000:0.0:1775764777.810021:0:11131:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978395136 00000100:00100000:0.0:1775764777.810024:0:11131:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978395136 00000100:00100000:2.0:1775764777.810027:0:5564:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c9880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+8:12181:x1862022978395648:12345-192.168.202.146@tcp:104: 00000100:00100000:0.0:1775764777.810035:0:11131:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978395264 00000100:00100000:0.0:1775764777.810038:0:11131:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978395264 00000100:00100000:0.0:1775764777.810045:0:11131:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978395392 00000100:00100000:0.0:1775764777.810047:0:11131:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978395392 00000100:00100000:0.0:1775764777.810055:0:11131:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978395520 00000100:00100000:0.0:1775764777.810057:0:11131:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978395520 00010000:00010000:2.0:1775764777.810057:0:5564:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005b4933ab/0x36f9bdc9e421c77d lrc: 2/0,0 mode: PR/PR res: [0x280000403:0x30:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5faf34 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1775764777.810071:0:11131:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24210 00000100:00100000:0.0:1775764777.810078:0:11131:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c8e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+10:12190:x1862022978394112:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764777.810088:0:11131:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000094d3ffa7/0x36f9bdc9e41cacb4 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x59:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eebcb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 180 lru_type: 1 00010000:00010000:2.0:1775764777.810129:0:5564:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1775764777.810137:0:5564:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c9880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+10:12181:x1862022978395648:12345-192.168.202.146@tcp:104: Request processed in 114us (308us total) trans 0 rc 0/0 00000100:00100000:2.0:1775764777.810143:0:5564:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 1131 00000100:00100000:2.0:1775764777.810175:0:5564:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 1132 00000100:00100000:2.0:1775764777.810184:0:5564:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9966933850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+9:12196:x1862022978396160:12345-192.168.202.146@tcp:104: 00000100:00100000:1.0:1775764777.810192:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24211 00010000:00010000:2.0:1775764777.810194:0:5564:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000aa9eb34d/0x36f9bdc9e421c179 lrc: 2/0,0 mode: PR/PR res: [0x280000403:0x31:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5f889f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000100:00100000:1.0:1775764777.810205:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0ca680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+9:12184:x1862022978394368:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764777.810216:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000009c24ad6b/0x36f9bdc9e41cac6e lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x58:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eeb23 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 73 lru_type: 1 00010000:00010000:0.0:1775764777.810444:0:11131:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:0.0:1775764777.810450:0:11131:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c8e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+9:12190:x1862022978394112:12345-192.168.202.146@tcp:104: Request processed in 373us (1765us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764777.810460:0:11131:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24210 00000100:00100000:0.0:1775764777.810475:0:11131:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24212 00000100:00100000:0.0:1775764777.810482:0:11131:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0ca300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+8:12188:x1862022978395008:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764777.810490:0:11131:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000047343c86/0x36f9bdc9e41cac1a lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x57:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eea58 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 0 00010000:00010000:0.0:1775764777.810520:0:11131:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:0.0:1775764777.810526:0:11131:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0ca300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+8:12188:x1862022978395008:12345-192.168.202.146@tcp:104: Request processed in 45us (854us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764777.810532:0:11131:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24212 00000100:00100000:0.0:1775764777.810543:0:11131:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24213 00000100:00100000:0.0:1775764777.810549:0:11131:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0c9f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:6578:x1862022978395136:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764777.810568:0:11131:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f82f5f70/0x36f9bdc9e41cac05 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x57:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eea4a expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 51 lru_type: 1 00010000:00010000:0.0:1775764777.810597:0:11131:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/4 locks to priority blp list, 4 blwis in pool 00000100:00100000:0.0:1775764777.810602:0:11131:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0c9f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:6578:x1862022978395136:12345-192.168.202.146@tcp:104: Request processed in 53us (892us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764777.810608:0:11131:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24213 00000100:00100000:0.0:1775764777.810619:0:11131:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24214 00000100:00100000:0.0:1775764777.810625:0:11131:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0cbb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:12051:x1862022978395264:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764777.810632:0:11131:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000009ff71b03/0x36f9bdc9e41cafd9 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x65:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef357 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 31 lru_type: 1 00010000:00010000:0.0:1775764777.810658:0:11131:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/5 locks to priority blp list, 5 blwis in pool 00000100:00100000:0.0:1775764777.810664:0:11131:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0cbb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:12051:x1862022978395264:12345-192.168.202.146@tcp:104: Request processed in 41us (932us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764777.810669:0:11131:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24214 00000100:00100000:0.0:1775764777.810680:0:11131:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24215 00000100:00100000:0.0:1775764777.810699:0:11131:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f0cb100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+7:6579:x1862022978395392:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764777.810707:0:11131:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000ceb22e30/0x36f9bdc9e41cabd4 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x56:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ee9b0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 0 00010000:00010000:0.0:1775764777.810735:0:11131:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/6 locks to priority blp list, 6 blwis in pool 00000100:00100000:0.0:1775764777.810767:0:11131:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0cb100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+7:6579:x1862022978395392:12345-192.168.202.146@tcp:104: Request processed in 70us (999us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764777.810772:0:11131:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24215 00000100:00100000:0.0:1775764777.810785:0:11131:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24216 00000100:00100000:0.0:1775764777.810791:0:11131:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9966bf6450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+6:16287:x1862022978395520:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764777.810798:0:11131:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005bcdc680/0x36f9bdc9e421c76f lrc: 2/0,0 mode: PR/PR res: [0x280000403:0x2f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5faf26 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764777.810816:0:11131:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/7 locks to priority blp list, 7 blwis in pool 00000100:00100000:0.0:1775764777.810820:0:11131:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9966bf6450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+6:16287:x1862022978395520:12345-192.168.202.146@tcp:104: Request processed in 31us (1011us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764777.810826:0:11131:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24216 00010000:00010000:0.0:1775764777.810844:0:11210:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 6 total in blp. (6 blwis in pool) 00010000:00010000:0.0:1775764777.810852:0:11210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005b4933ab/0x36f9bdc9e421c77d lrc: 2/0,0 mode: PR/PR res: [0x280000403:0x30:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5faf34 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764777.810861:0:11210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99821cde00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:0.0:1775764777.810867:0:11210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005b4933ab/0x36f9bdc9e421c77d lrc: 3/0,0 mode: PR/PR res: [0x280000403:0x30:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x428400010000 nid: local remote: 0x2aac6a50e5faf34 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764777.811188:0:11210:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005b4933ab/0x36f9bdc9e421c77d lrc: 2/0,0 mode: --/PR res: [0x280000403:0x30:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5faf34 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764777.811198:0:11210:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764777.811218:0:11210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005b4933ab/0x36f9bdc9e421c77d lrc: 1/0,0 mode: --/PR res: [0x280000403:0x30:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5faf34 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764777.811225:0:11210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005b4933ab/0x36f9bdc9e421c77d lrc: 0/0,0 mode: --/PR res: [0x280000403:0x30:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5faf34 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764777.811249:0:11210:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 5 total in blp. (5 blwis in pool) 00010000:00010000:0.0:1775764777.811253:0:11210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000094d3ffa7/0x36f9bdc9e41cacb4 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x59:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eebcb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 180 lru_type: 1 00010000:00010000:0.0:1775764777.811259:0:11210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970adfa00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:0.0:1775764777.811263:0:11210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000094d3ffa7/0x36f9bdc9e41cacb4 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x59:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4eebcb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 180 lru_type: 1 00010000:00010000:0.0:1775764777.811349:0:11210:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000094d3ffa7/0x36f9bdc9e41cacb4 lrc: 2/0,0 mode: --/PW res: [0x280000403:0x59:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eebcb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 180 lru_type: 1 00010000:00010000:0.0:1775764777.811357:0:11210:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764777.811363:0:11210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000094d3ffa7/0x36f9bdc9e41cacb4 lrc: 1/0,0 mode: --/PW res: [0x280000403:0x59:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eebcb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 180 lru_type: 1 00010000:00010000:0.0:1775764777.811375:0:11210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000094d3ffa7/0x36f9bdc9e41cacb4 lrc: 0/0,0 mode: --/PW res: [0x280000403:0x59:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eebcb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 180 lru_type: 1 00010000:00010000:0.0:1775764777.811389:0:11210:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 4 total in blp. (4 blwis in pool) 00010000:00010000:0.0:1775764777.811392:0:11210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000047343c86/0x36f9bdc9e41cac1a lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x57:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eea58 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 0 00010000:00010000:0.0:1775764777.811399:0:11210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970adf600 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:0.0:1775764777.811403:0:11210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000047343c86/0x36f9bdc9e41cac1a lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x57:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4eea58 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 0 00010000:00010000:0.0:1775764777.811446:0:11210:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000047343c86/0x36f9bdc9e41cac1a lrc: 2/0,0 mode: --/PW res: [0x280000403:0x57:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eea58 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 0 00010000:00010000:0.0:1775764777.811453:0:11210:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764777.811459:0:11210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000047343c86/0x36f9bdc9e41cac1a lrc: 1/0,0 mode: --/PW res: [0x280000403:0x57:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eea58 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 0 00010000:00010000:0.0:1775764777.811465:0:11210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000047343c86/0x36f9bdc9e41cac1a lrc: 0/0,0 mode: --/PW res: [0x280000403:0x57:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eea58 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 0 00010000:00010000:0.0:1775764777.811479:0:11210:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 3 total in blp. (3 blwis in pool) 00010000:00010000:0.0:1775764777.811482:0:11210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f82f5f70/0x36f9bdc9e41cac05 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x57:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eea4a expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 51 lru_type: 1 00010000:00010000:0.0:1775764777.811494:0:11210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970641e00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:0.0:1775764777.811497:0:11210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f82f5f70/0x36f9bdc9e41cac05 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x57:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4eea4a expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 51 lru_type: 1 00010000:00010000:0.0:1775764777.811544:0:11210:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f82f5f70/0x36f9bdc9e41cac05 lrc: 2/0,0 mode: --/PW res: [0x240000403:0x57:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eea4a expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 51 lru_type: 1 00010000:00010000:0.0:1775764777.811551:0:11210:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764777.811564:0:11210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f82f5f70/0x36f9bdc9e41cac05 lrc: 1/0,0 mode: --/PW res: [0x240000403:0x57:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eea4a expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 51 lru_type: 1 00010000:00010000:0.0:1775764777.811572:0:11210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f82f5f70/0x36f9bdc9e41cac05 lrc: 0/0,0 mode: --/PW res: [0x240000403:0x57:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eea4a expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 51 lru_type: 1 00010000:00010000:0.0:1775764777.811586:0:11210:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:0.0:1775764777.811589:0:11210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000009ff71b03/0x36f9bdc9e41cafd9 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x65:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ef357 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 31 lru_type: 1 00010000:00010000:0.0:1775764777.811595:0:11210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9941480600 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:0.0:1775764777.811598:0:11210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000009ff71b03/0x36f9bdc9e41cafd9 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x65:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ef357 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 31 lru_type: 1 00010000:00010000:0.0:1775764777.811628:0:11210:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000009ff71b03/0x36f9bdc9e41cafd9 lrc: 2/0,0 mode: --/PW res: [0x240000403:0x65:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef357 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 31 lru_type: 1 00010000:00010000:0.0:1775764777.811641:0:11210:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764777.811646:0:11210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000009ff71b03/0x36f9bdc9e41cafd9 lrc: 1/0,0 mode: --/PW res: [0x240000403:0x65:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef357 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 31 lru_type: 1 00010000:00010000:0.0:1775764777.811652:0:11210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000009ff71b03/0x36f9bdc9e41cafd9 lrc: 0/0,0 mode: --/PW res: [0x240000403:0x65:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ef357 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 31 lru_type: 1 00010000:00010000:0.0:1775764777.811665:0:11210:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1775764777.811669:0:11210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000ceb22e30/0x36f9bdc9e41cabd4 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x56:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ee9b0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 0 00010000:00010000:0.0:1775764777.812466:0:11210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9971227a00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:0.0:1775764777.812472:0:11210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000ceb22e30/0x36f9bdc9e41cabd4 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x56:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ee9b0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 0 00010000:00010000:0.0:1775764777.812561:0:11210:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000ceb22e30/0x36f9bdc9e41cabd4 lrc: 2/0,0 mode: --/PW res: [0x280000403:0x56:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee9b0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 0 00010000:00010000:0.0:1775764777.812569:0:11210:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764777.812575:0:11210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000ceb22e30/0x36f9bdc9e41cabd4 lrc: 1/0,0 mode: --/PW res: [0x280000403:0x56:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee9b0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 0 00010000:00010000:0.0:1775764777.812582:0:11210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000ceb22e30/0x36f9bdc9e41cabd4 lrc: 0/0,0 mode: --/PW res: [0x280000403:0x56:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee9b0 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 0 00010000:00010000:0.0:1775764777.812598:0:11210:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1775764777.812602:0:11210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005bcdc680/0x36f9bdc9e421c76f lrc: 2/0,0 mode: PR/PR res: [0x280000403:0x2f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5faf26 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764777.812609:0:11210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99706eb400 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:0.0:1775764777.812613:0:11210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005bcdc680/0x36f9bdc9e421c76f lrc: 3/0,0 mode: PR/PR res: [0x280000403:0x2f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x428400010000 nid: local remote: 0x2aac6a50e5faf26 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764777.812646:0:11210:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005bcdc680/0x36f9bdc9e421c76f lrc: 2/0,0 mode: --/PR res: [0x280000403:0x2f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5faf26 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764777.812652:0:11210:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764777.812656:0:11210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005bcdc680/0x36f9bdc9e421c76f lrc: 1/0,0 mode: --/PR res: [0x280000403:0x2f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5faf26 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764777.812663:0:11210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000005bcdc680/0x36f9bdc9e421c76f lrc: 0/0,0 mode: --/PR res: [0x280000403:0x2f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5faf26 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764777.814006:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764777.814011:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764777.814014:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764777.814026:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764777.814031:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764777.814054:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f0cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967016832:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764777.814133:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967017088:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764777.814153:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967017344:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764777.814174:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967017600:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764777.814196:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [1->0] 00000100:00100000:0.0:1775764777.814203:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967016960:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764777.814221:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967017216:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764777.814235:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967017472:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764777.814287:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764777.814397:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00010000:00010000:2.0:1775764777.815829:0:5564:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1775764777.815846:0:5564:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9966933850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+5:12196:x1862022978396160:12345-192.168.202.146@tcp:104: Request processed in 5658us (5982us total) trans 0 rc 0/0 00000100:00100000:2.0:1775764777.815859:0:5564:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 1132 00010000:00010000:2.0:1775764777.815938:0:19800:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764777.815947:0:19800:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000aa9eb34d/0x36f9bdc9e421c179 lrc: 2/0,0 mode: PR/PR res: [0x280000403:0x31:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5f889f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764777.815959:0:19800:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f994ed8cc00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:2.0:1775764777.815966:0:19800:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000aa9eb34d/0x36f9bdc9e421c179 lrc: 3/0,0 mode: PR/PR res: [0x280000403:0x31:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x2aac6a50e5f889f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764777.816358:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764777.816373:0:11131:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978396288 00000100:00100000:0.0:1775764777.816380:0:11131:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978396288 00000100:00100000:0.0:1775764777.816408:0:11131:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978396416 00000100:00100000:0.0:1775764777.816412:0:11131:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978396416 00000100:00100000:0.0:1775764777.816428:0:11131:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24217 00000100:00100000:0.0:1775764777.816437:0:11131:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970a75c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:12180:x1862022978396288:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764777.816452:0:11131:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a41d1221/0x36f9bdc9e421a84a lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x655:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f3ccc expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 0 00010000:00010000:0.0:1775764777.816514:0:11131:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764777.816519:0:11131:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970a75c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:12180:x1862022978396288:12345-192.168.202.146@tcp:104: Request processed in 84us (2234us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764777.816525:0:11131:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24217 00000100:00100000:0.0:1775764777.816538:0:11131:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24218 00000100:00100000:0.0:1775764777.816544:0:11131:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970a77480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:8216:x1862022978396416:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764777.816552:0:11131:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003cb7abbf/0x36f9bdc9e41cab72 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x55:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ee8de expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 83 lru_type: 1 00010000:00010000:1.0:1775764777.817285:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1775764777.817298:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f0ca680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:12184:x1862022978394368:12345-192.168.202.146@tcp:104: Request processed in 7095us (7644us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764777.817306:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24211 00010000:00010000:1.0:1775764777.817390:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1775764777.817396:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a41d1221/0x36f9bdc9e421a84a lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x655:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5f3ccc expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 0 00010000:00010000:1.0:1775764777.817406:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970973000 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:1.0:1775764777.817411:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a41d1221/0x36f9bdc9e421a84a lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x655:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e5f3ccc expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 0 00010000:00010000:1.0:1775764777.817493:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a41d1221/0x36f9bdc9e421a84a lrc: 2/0,0 mode: --/PW res: [0x240000403:0x655:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f3ccc expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 0 00010000:00010000:1.0:1775764777.817505:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764777.817525:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a41d1221/0x36f9bdc9e421a84a lrc: 1/0,0 mode: --/PW res: [0x240000403:0x655:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f3ccc expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 0 00010000:00010000:1.0:1775764777.817533:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000a41d1221/0x36f9bdc9e421a84a lrc: 0/0,0 mode: --/PW res: [0x240000403:0x655:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5f3ccc expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 0 00010000:00010000:1.0:1775764777.817564:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764777.817569:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000009c24ad6b/0x36f9bdc9e41cac6e lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x58:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4eeb23 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 73 lru_type: 1 00010000:00010000:1.0:1775764777.817576:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970adfe00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:1.0:1775764777.817579:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000009c24ad6b/0x36f9bdc9e41cac6e lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x58:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4eeb23 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 73 lru_type: 1 00010000:00010000:1.0:1775764777.817613:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000009c24ad6b/0x36f9bdc9e41cac6e lrc: 2/0,0 mode: --/PW res: [0x280000403:0x58:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eeb23 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 73 lru_type: 1 00010000:00010000:1.0:1775764777.817620:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764777.817626:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000009c24ad6b/0x36f9bdc9e41cac6e lrc: 1/0,0 mode: --/PW res: [0x280000403:0x58:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eeb23 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 73 lru_type: 1 00010000:00010000:1.0:1775764777.817632:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 000000009c24ad6b/0x36f9bdc9e41cac6e lrc: 0/0,0 mode: --/PW res: [0x280000403:0x58:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4eeb23 expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 73 lru_type: 1 00010000:00010000:1.0:1775764777.817648:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764777.817650:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764777.817652:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764777.817660:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764777.817665:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764777.817791:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967017728:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764777.817948:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967016704:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764777.817960:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0c9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967016192:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764777.817979:0:2380:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1775764777.817985:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967017856:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1775764777.818265:0:19800:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000aa9eb34d/0x36f9bdc9e421c179 lrc: 2/0,0 mode: --/PR res: [0x280000403:0x31:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5f889f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764777.818274:0:19800:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764777.818295:0:19800:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000aa9eb34d/0x36f9bdc9e421c179 lrc: 1/0,0 mode: --/PR res: [0x280000403:0x31:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5f889f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764777.818302:0:19800:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000aa9eb34d/0x36f9bdc9e421c179 lrc: 0/0,0 mode: --/PR res: [0x280000403:0x31:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5f889f expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764777.818320:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764777.818323:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764777.818324:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764777.818337:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967017984:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1775764777.821589:0:11131:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764777.821603:0:11131:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970a77480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:8216:x1862022978396416:12345-192.168.202.146@tcp:104: Request processed in 5054us (7200us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764777.821615:0:11131:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24218 00010000:00010000:0.0:1775764777.821656:0:11138:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1775764777.821663:0:11138:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003cb7abbf/0x36f9bdc9e41cab72 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x55:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e4ee8de expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 83 lru_type: 1 00010000:00010000:0.0:1775764777.821672:0:11138:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f994f3d9200 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:0.0:1775764777.821679:0:11138:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003cb7abbf/0x36f9bdc9e41cab72 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x55:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e4ee8de expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 83 lru_type: 1 00010000:00010000:0.0:1775764777.821794:0:11138:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003cb7abbf/0x36f9bdc9e41cab72 lrc: 2/0,0 mode: --/PW res: [0x240000403:0x55:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee8de expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 83 lru_type: 1 00010000:00010000:0.0:1775764777.821803:0:11138:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764777.821822:0:11138:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003cb7abbf/0x36f9bdc9e41cab72 lrc: 1/0,0 mode: --/PW res: [0x240000403:0x55:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee8de expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 83 lru_type: 1 00010000:00010000:0.0:1775764777.821830:0:11138:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000003cb7abbf/0x36f9bdc9e41cab72 lrc: 0/0,0 mode: --/PW res: [0x240000403:0x55:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 61440->65535) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e4ee8de expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 83 lru_type: 1 00010000:00010000:0.0:1775764777.821853:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764777.821855:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764777.821857:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764777.822566:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764777.823629:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1775764777.823661:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967018112:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764777.833651:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764777.833864:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967016960:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764777.833891:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967017216:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764777.833900:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967017472:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764777.833908:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f0cad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967016832:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764777.833915:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967017088:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764777.833921:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967017344:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764777.833928:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967017600:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764777.833974:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978396672 00000100:00100000:1.0:1775764777.833982:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978396672 00000100:00100000:1.0:1775764777.834030:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24219 00000100:00100000:1.0:1775764777.834039:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e5880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:12039:x1862022978396672:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764777.834056:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000015339cf1/0x36f9bdc9e421c7a7 lrc: 2/0,0 mode: PR/PR res: [0x280000403:0x32:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5fb022 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764777.834128:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764777.834134:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e5880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:12039:x1862022978396672:12345-192.168.202.146@tcp:104: Request processed in 97us (492us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764777.834142:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24219 00010000:00010000:1.0:1775764777.834166:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764777.834172:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000015339cf1/0x36f9bdc9e421c7a7 lrc: 2/0,0 mode: PR/PR res: [0x280000403:0x32:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5fb022 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764777.834181:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99706ebc00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:1.0:1775764777.834187:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000015339cf1/0x36f9bdc9e421c7a7 lrc: 3/0,0 mode: PR/PR res: [0x280000403:0x32:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x428400010000 nid: local remote: 0x2aac6a50e5fb022 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764777.834267:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000015339cf1/0x36f9bdc9e421c7a7 lrc: 2/0,0 mode: --/PR res: [0x280000403:0x32:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5fb022 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764777.834275:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764777.834293:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000015339cf1/0x36f9bdc9e421c7a7 lrc: 1/0,0 mode: --/PR res: [0x280000403:0x32:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5fb022 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764777.834302:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000015339cf1/0x36f9bdc9e421c7a7 lrc: 0/0,0 mode: --/PR res: [0x280000403:0x32:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5fb022 expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764777.834327:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764777.834329:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764777.834331:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764777.834340:0:2380:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1775764777.834353:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967018240:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764777.851404:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764777.851443:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978396928 00000100:00100000:1.0:1775764777.851453:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978396928 00000100:00100000:1.0:1775764777.851504:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24220 00000100:00100000:1.0:1775764777.851515:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e6d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:16465:x1862022978396928:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764777.851534:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000031070867/0x36f9bdc9e421c164 lrc: 2/0,0 mode: PR/PR res: [0x280000403:0x33:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5f888a expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764777.851619:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764777.851625:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e6d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:16465:x1862022978396928:12345-192.168.202.146@tcp:104: Request processed in 113us (232us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764777.851632:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24220 00010000:00010000:2.0:1775764777.851720:0:5572:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764777.851730:0:5572:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000031070867/0x36f9bdc9e421c164 lrc: 2/0,0 mode: PR/PR res: [0x280000403:0x33:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5f888a expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764777.851751:0:5572:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f994f3d9400 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:2.0:1775764777.851757:0:5572:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000031070867/0x36f9bdc9e421c164 lrc: 3/0,0 mode: PR/PR res: [0x280000403:0x33:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x428400010000 nid: local remote: 0x2aac6a50e5f888a expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764777.851840:0:5572:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000031070867/0x36f9bdc9e421c164 lrc: 2/0,0 mode: --/PR res: [0x280000403:0x33:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5f888a expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764777.851849:0:5572:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764777.851878:0:5572:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000031070867/0x36f9bdc9e421c164 lrc: 1/0,0 mode: --/PR res: [0x280000403:0x33:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5f888a expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764777.851887:0:5572:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000031070867/0x36f9bdc9e421c164 lrc: 0/0,0 mode: --/PR res: [0x280000403:0x33:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4096->8191) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5f888a expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764777.851910:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764777.851912:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764777.851914:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764777.851927:0:2382:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1775764777.851941:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967018368:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764777.863502:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967018112:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764777.863613:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967018240:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764777.863625:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967017856:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764777.863634:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967017728:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764777.864235:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967017984:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764777.878975:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967018368:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1775764778.516306:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764779.527036:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764779.723996:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2382:1862022967018496:192.168.202.146@tcp:400:kworker.0 00000100:00100000:2.0:1775764779.724091:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967018752:192.168.202.146@tcp:400:kworker.0 00000100:00100000:2.0:1775764779.724127:0:2382:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:2.0:1775764779.724135:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967018624:192.168.202.146@tcp:400:kworker.0 00000100:00100000:2.0:1775764779.724159:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967018880:192.168.202.146@tcp:400:kworker.0 00000100:00100000:2.0:1775764779.744008:0:2382:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8f99523ee800: committing for last_committed 4294997596 gen 1 00000100:00100000:2.0:1775764779.744621:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967018624:192.168.202.146@tcp:400:kworker.0 00000100:00100000:2.0:1775764779.744644:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967018880:192.168.202.146@tcp:400:kworker.0 00000100:00100000:2.0:1775764779.744654:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2382:1862022967018496:192.168.202.146@tcp:400:kworker.0 00000100:00100000:2.0:1775764779.744665:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967018752:192.168.202.146@tcp:400:kworker.0 00010000:00010000:1.0:1775764780.559220:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764781.581229:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764782.601854:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764782.703475:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764782.703504:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764782.703540:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764782.703562:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764782.703578:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764782.703593:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764782.703613:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764782.703641:0:5561:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1775764782.703659:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978398848 00000100:00100000:0.0:1775764782.703670:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978398848 00000100:00100000:0.0:1775764782.703716:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978399104 00000100:00100000:0.0:1775764782.703719:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978399104 00000100:00100000:0.0:1775764782.703728:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978399232 00000100:00100000:0.0:1775764782.703730:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978399232 00000100:00100000:0.0:1775764782.703777:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24221 00000100:00100000:0.0:1775764782.703787:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970a77800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+6:6578:x1862022978398848:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764782.703810:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ae8788d7/0x36f9bdc9e421c768 lrc: 2/0,0 mode: PR/PR res: [0x240000403:0x2e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5faf1f expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764782.703887:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764782.703894:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970a77800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+6:6578:x1862022978398848:12345-192.168.202.146@tcp:104: Request processed in 109us (429us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764782.703901:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24221 00000100:00100000:0.0:1775764782.703917:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24222 00000100:00100000:0.0:1775764782.703924:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970a76a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:12180:x1862022978399104:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764782.703933:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f9afe1c6/0x36f9bdc9e4211ece lrc: 2/0,0 mode: PR/PR res: [0x240000403:0x31:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5d76f9 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764782.703960:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:0.0:1775764782.703966:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970a76a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:12180:x1862022978399104:12345-192.168.202.146@tcp:104: Request processed in 42us (461us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764782.703972:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24222 00000100:00100000:0.0:1775764782.703982:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24223 00000100:00100000:0.0:1775764782.703988:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9966bf4050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:12182:x1862022978399232:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764782.703996:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c776 lrc: 2/0,0 mode: PR/PR res: [0x240000403:0x30:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5faf2d expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764782.704022:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:0.0:1775764782.704027:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9966bf4050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:12182:x1862022978399232:12345-192.168.202.146@tcp:104: Request processed in 40us (487us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764782.704033:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24223 00010000:00010000:0.0:1775764782.704173:0:5572:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:0.0:1775764782.704179:0:5572:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ae8788d7/0x36f9bdc9e421c768 lrc: 2/0,0 mode: PR/PR res: [0x240000403:0x2e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5faf1f expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764782.704188:0:5572:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99706eb800 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:0.0:1775764782.704194:0:5572:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ae8788d7/0x36f9bdc9e421c768 lrc: 3/0,0 mode: PR/PR res: [0x240000403:0x2e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x428400010000 nid: local remote: 0x2aac6a50e5faf1f expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764782.704275:0:5572:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ae8788d7/0x36f9bdc9e421c768 lrc: 2/0,0 mode: --/PR res: [0x240000403:0x2e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5faf1f expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764782.704284:0:5572:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764782.704319:0:5572:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ae8788d7/0x36f9bdc9e421c768 lrc: 1/0,0 mode: --/PR res: [0x240000403:0x2e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5faf1f expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764782.704328:0:5572:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000ae8788d7/0x36f9bdc9e421c768 lrc: 0/0,0 mode: --/PR res: [0x240000403:0x2e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5faf1f expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764782.704348:0:5572:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1775764782.704352:0:5572:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f9afe1c6/0x36f9bdc9e4211ece lrc: 2/0,0 mode: PR/PR res: [0x240000403:0x31:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5d76f9 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764782.704359:0:5572:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f994f3b9a00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:0.0:1775764782.704363:0:5572:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f9afe1c6/0x36f9bdc9e4211ece lrc: 3/0,0 mode: PR/PR res: [0x240000403:0x31:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x428400010000 nid: local remote: 0x2aac6a50e5d76f9 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764782.704396:0:5572:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f9afe1c6/0x36f9bdc9e4211ece lrc: 2/0,0 mode: --/PR res: [0x240000403:0x31:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5d76f9 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764782.704403:0:5572:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764782.704409:0:5572:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f9afe1c6/0x36f9bdc9e4211ece lrc: 1/0,0 mode: --/PR res: [0x240000403:0x31:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5d76f9 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764782.704416:0:5572:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000f9afe1c6/0x36f9bdc9e4211ece lrc: 0/0,0 mode: --/PR res: [0x240000403:0x31:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8192->12287) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5d76f9 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764782.704428:0:5572:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1775764782.704431:0:5572:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c776 lrc: 2/0,0 mode: PR/PR res: [0x240000403:0x30:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5faf2d expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764782.704438:0:5572:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99821cc800 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:0.0:1775764782.704442:0:5572:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c776 lrc: 3/0,0 mode: PR/PR res: [0x240000403:0x30:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x428400010000 nid: local remote: 0x2aac6a50e5faf2d expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764782.704473:0:5572:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c776 lrc: 2/0,0 mode: --/PR res: [0x240000403:0x30:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5faf2d expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764782.704480:0:5572:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764782.704485:0:5572:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c776 lrc: 1/0,0 mode: --/PR res: [0x240000403:0x30:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5faf2d expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764782.704492:0:5572:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c776 lrc: 0/0,0 mode: --/PR res: [0x240000403:0x30:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5faf2d expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764782.704505:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764782.704507:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764782.704522:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764782.704529:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764782.704533:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764782.704546:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967019008:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764782.704607:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967019264:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764782.704630:0:2380:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1775764782.704635:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967019136:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764782.704896:0:5564:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978398976 00000100:00100000:2.0:1775764782.704902:0:5564:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978398976 00000100:00100000:2.0:1775764782.704926:0:5564:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978399360 00000100:00100000:2.0:1775764782.704929:0:5564:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978399360 00000100:00100000:2.0:1775764782.704937:0:5564:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978399488 00000100:00100000:2.0:1775764782.704939:0:5564:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978399488 00000100:00100000:2.0:1775764782.704947:0:5564:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978399616 00000100:00100000:2.0:1775764782.704949:0:5564:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978399616 00000100:00100000:2.0:1775764782.704959:0:5564:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 1133 00000100:00100000:2.0:1775764782.704965:0:5564:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970a76680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+7:6579:x1862022978398976:12345-192.168.202.146@tcp:104: 00010000:00010000:2.0:1775764782.704974:0:5564:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4f14cf5/0x36f9bdc9e4211ec0 lrc: 2/0,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1232896->1236991) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5d76eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 0 00010000:00010000:2.0:1775764782.705022:0:5564:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1775764782.705029:0:5564:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970a76680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+7:6579:x1862022978398976:12345-192.168.202.146@tcp:104: Request processed in 63us (1466us total) trans 0 rc 0/0 00000100:00100000:2.0:1775764782.705034:0:5564:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 1133 00000100:00100000:2.0:1775764782.705047:0:5564:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 1134 00000100:00100000:2.0:1775764782.705052:0:5564:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970a75f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+6:6579:x1862022978399360:12345-192.168.202.146@tcp:104: 00010000:00010000:2.0:1775764782.705060:0:5564:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 2/0,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 0 00010000:00010000:2.0:1775764782.705086:0:5564:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:2.0:1775764782.705090:0:5564:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970a75f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+6:6579:x1862022978399360:12345-192.168.202.146@tcp:104: Request processed in 38us (1512us total) trans 0 rc 0/0 00000100:00100000:2.0:1775764782.705095:0:5564:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 1134 00000100:00100000:2.0:1775764782.705106:0:5564:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 1135 00000100:00100000:2.0:1775764782.705111:0:5564:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970a74a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+5:12188:x1862022978399488:12345-192.168.202.146@tcp:104: 00010000:00010000:2.0:1775764782.705118:0:5564:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000062300bdc/0x36f9bdc9e41c9bab lrc: 2/0,0 mode: PR/PR res: [0x240000403:0x2f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8388607) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e4ebe4b expref: -99 pid: 19698 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:2.0:1775764782.705140:0:5564:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:2.0:1775764782.705145:0:5564:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970a74a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+5:12188:x1862022978399488:12345-192.168.202.146@tcp:104: Request processed in 35us (1552us total) trans 0 rc 0/0 00000100:00100000:2.0:1775764782.705151:0:5564:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 1135 00000100:00100000:2.0:1775764782.705160:0:5564:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 1136 00000100:00100000:2.0:1775764782.705167:0:5564:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970a74000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:12196:x1862022978399616:12345-192.168.202.146@tcp:104: 00010000:00010000:2.0:1775764782.705173:0:5564:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c8654ade/0x36f9bdc9e421c7ae lrc: 2/0,0 mode: PR/PR res: [0x240000403:0x33:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8388607) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5fb01b expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764782.705198:0:5564:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/4 locks to priority blp list, 4 blwis in pool 00000100:00100000:2.0:1775764782.705203:0:5564:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970a74000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:12196:x1862022978399616:12345-192.168.202.146@tcp:104: Request processed in 37us (1589us total) trans 0 rc 0/0 00000100:00100000:2.0:1775764782.705208:0:5564:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 1136 00010000:00010000:2.0:1775764782.705227:0:11210:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 3 total in blp. (3 blwis in pool) 00010000:00010000:2.0:1775764782.705231:0:11210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4f14cf5/0x36f9bdc9e4211ec0 lrc: 2/0,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1232896->1236991) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5d76eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 0 00010000:00010000:2.0:1775764782.705238:0:11210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970925e00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:2.0:1775764782.705243:0:11210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4f14cf5/0x36f9bdc9e4211ec0 lrc: 3/0,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1232896->1236991) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e5d76eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 0 00010000:00010000:0.0:1775764782.705438:0:5572:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:0.0:1775764782.705443:0:5572:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 2/0,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 4 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x420000030000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 0 00010000:00010000:0.0:1775764782.705450:0:5572:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970924c00 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:0.0:1775764782.705454:0:5572:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 3/0,0 mode: PR/PR res: [0x240000403:0x32:0x0].0x0 rrc: 4 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x428400030000 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 0 00010000:00010000:0.0:1775764782.705494:0:5572:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 2/0,0 mode: --/PR res: [0x240000403:0x32:0x0].0x0 rrc: 4 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 0 00010000:00010000:0.0:1775764782.705500:0:5572:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764782.705525:0:5572:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 1/0,0 mode: --/PR res: [0x240000403:0x32:0x0].0x0 rrc: 4 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 0 00010000:00010000:0.0:1775764782.705532:0:5572:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000000506430b/0x36f9bdc9e4211eb9 lrc: 0/0,0 mode: --/PR res: [0x240000403:0x32:0x0].0x0 rrc: 4 type: EXT [0->1232895] (req 1060864->1064959) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5d76e4 expref: -99 pid: 19724 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 0 00010000:00010000:0.0:1775764782.705544:0:5572:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1775764782.705547:0:5572:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000062300bdc/0x36f9bdc9e41c9bab lrc: 2/0,0 mode: PR/PR res: [0x240000403:0x2f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8388607) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e4ebe4b expref: -99 pid: 19698 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:0.0:1775764782.705553:0:5572:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99435d5200 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:0.0:1775764782.705557:0:5572:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000062300bdc/0x36f9bdc9e41c9bab lrc: 3/0,0 mode: PR/PR res: [0x240000403:0x2f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8388607) gid 0 flags: 0x428400010000 nid: local remote: 0x2aac6a50e4ebe4b expref: -99 pid: 19698 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:0.0:1775764782.705589:0:5572:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000062300bdc/0x36f9bdc9e41c9bab lrc: 2/0,0 mode: --/PR res: [0x240000403:0x2f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8388607) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e4ebe4b expref: -99 pid: 19698 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:0.0:1775764782.705595:0:5572:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764782.705601:0:5572:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000062300bdc/0x36f9bdc9e41c9bab lrc: 1/0,0 mode: --/PR res: [0x240000403:0x2f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8388607) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e4ebe4b expref: -99 pid: 19698 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:0.0:1775764782.705607:0:5572:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 0000000062300bdc/0x36f9bdc9e41c9bab lrc: 0/0,0 mode: --/PR res: [0x240000403:0x2f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8388607) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e4ebe4b expref: -99 pid: 19698 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:0.0:1775764782.705620:0:5572:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1775764782.705622:0:5572:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c8654ade/0x36f9bdc9e421c7ae lrc: 2/0,0 mode: PR/PR res: [0x240000403:0x33:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8388607) gid 0 flags: 0x420000010000 nid: local remote: 0x2aac6a50e5fb01b expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764782.705628:0:5572:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99706ea800 already unused, calling callback (ffffffffc12102c0) 00010000:00010000:0.0:1775764782.705631:0:5572:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c8654ade/0x36f9bdc9e421c7ae lrc: 3/0,0 mode: PR/PR res: [0x240000403:0x33:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8388607) gid 0 flags: 0x428400010000 nid: local remote: 0x2aac6a50e5fb01b expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764782.705658:0:5572:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c8654ade/0x36f9bdc9e421c7ae lrc: 2/0,0 mode: --/PR res: [0x240000403:0x33:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8388607) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5fb01b expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764782.705664:0:5572:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764782.705668:0:5572:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c8654ade/0x36f9bdc9e421c7ae lrc: 1/0,0 mode: --/PR res: [0x240000403:0x33:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8388607) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5fb01b expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764782.705674:0:5572:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c8654ade/0x36f9bdc9e421c7ae lrc: 0/0,0 mode: --/PR res: [0x240000403:0x33:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8388607) gid 0 flags: 0x4c29400010020 nid: local remote: 0x2aac6a50e5fb01b expref: -99 pid: 19896 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764782.705685:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764782.705687:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764782.705689:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764782.705699:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967019520:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764782.705762:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967019776:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764782.705785:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1775764782.705789:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967019648:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1775764782.705902:0:11210:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4f14cf5/0x36f9bdc9e4211ec0 lrc: 2/0,0 mode: --/PR res: [0x240000403:0x32:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1232896->1236991) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5d76eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 0 00010000:00010000:2.0:1775764782.705912:0:11210:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764782.705928:0:11210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4f14cf5/0x36f9bdc9e4211ec0 lrc: 1/0,0 mode: --/PR res: [0x240000403:0x32:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1232896->1236991) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5d76eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 0 00010000:00010000:2.0:1775764782.705935:0:11210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000c4f14cf5/0x36f9bdc9e4211ec0 lrc: 0/0,0 mode: --/PR res: [0x240000403:0x32:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1232896->1236991) gid 0 flags: 0x4c29400030020 nid: local remote: 0x2aac6a50e5d76eb expref: -99 pid: 19723 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 0 00010000:00010000:2.0:1775764782.705950:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764782.705952:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764782.705954:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764782.705960:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764782.705969:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967019392:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1775764782.707676:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764782.721168:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967019136:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764782.721181:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967019008:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764782.721188:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967019264:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764782.721221:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967019648:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764782.721228:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967019520:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764782.721235:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967019776:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764782.721302:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967019392:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1775764783.624634:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1775764783.966580:0:20459:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-pfl test 9: Replay layout extend object instantiation ========================================================== 15:59:41 (1775764781) 00000080:00200000:0.0:1775764784.228693:0:20469:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:0.0:1775764784.228724:0:20469:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764784.228812:0:20469:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cde00/0x36f9bdc9e421c7e6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 20469 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764784.228824:0:20469:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cde00/0x36f9bdc9e421c7e6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 20469 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764784.228836:0:20469:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cde00/0x36f9bdc9e421c7e6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 20469 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764784.228866:0:20469:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job mkdir:3f83546c-01a9-4156-8a97-44c446245d3e:20469:1862022967019904:192.168.202.146@tcp:101:mkdir.0 00000100:00100000:1.0:1775764784.231006:0:20469:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960d31300 going to sleep for 11 seconds 00000100:00100000:1.0:1775764784.243735:0:20469:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job mkdir:3f83546c-01a9-4156-8a97-44c446245d3e:20469:1862022967019904:192.168.202.146@tcp:101:mkdir.0 00010000:00010000:1.0:1775764784.243788:0:20469:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cde00/0x36f9bdc9e421c7e6 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb298 expref: -99 pid: 20469 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.243802:0:20469:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cde00/0x36f9bdc9e421c7e6 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb298 expref: -99 pid: 20469 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.243815:0:20469:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cde00/0x36f9bdc9e421c7e6 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb298 expref: -99 pid: 20469 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.243827:0:20469:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cde00/0x36f9bdc9e421c7e6 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb298 expref: -99 pid: 20469 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.243842:0:20469:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cde00/0x36f9bdc9e421c7e6 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb298 expref: -99 pid: 20469 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.243852:0:20469:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cde00/0x36f9bdc9e421c7e6 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb298 expref: -99 pid: 20469 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.243860:0:20469:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cde00/0x36f9bdc9e421c7e6 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb298 expref: -99 pid: 20469 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764784.243872:0:20469:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970a75180 x1862022967019904/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764800 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'mkdir.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764784.243886:0:20469:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cde00/0x36f9bdc9e421c7e6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb298 expref: -99 pid: 20469 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.243901:0:20469:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1775764784.243908:0:20469:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1775764784.243939:0:20469:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 8704 00000080:00010000:1.0:1775764784.243944:0:20469:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c7e6 00000080:00010000:1.0:1775764784.243950:0:20469:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764784.243953:0:20469:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c7e6 from it ffffafbac19c7be0 00010000:00010000:1.0:1775764784.243957:0:20469:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cde00/0x36f9bdc9e421c7e6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb298 expref: -99 pid: 20469 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.243964:0:20469:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cde00/0x36f9bdc9e421c7e6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb298 expref: -99 pid: 20469 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764784.244001:0:20469:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764784.244023:0:20469:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=d9.sanity-pfl, dir=[0x200000007:0x1:0x0](ffff8f9952d121c8), flags=1536 00000080:00200000:1.0:1775764784.244028:0:20469:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764784.244043:0:20469:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764784.244052:0:20469:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cde00/0x36f9bdc9e421c7e6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb298 expref: -99 pid: 20469 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764784.244061:0:20469:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cde00/0x36f9bdc9e421c7e6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb298 expref: -99 pid: 20469 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764784.244067:0:20469:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764784.244071:0:20469:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764784.244073:0:20469:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c7e6 00000080:00010000:1.0:1775764784.244077:0:20469:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c7e6 from it ffffafbac19c7ce0 00010000:00010000:1.0:1775764784.244080:0:20469:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cde00/0x36f9bdc9e421c7e6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb298 expref: -99 pid: 20469 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764784.244086:0:20469:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cde00/0x36f9bdc9e421c7e6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb298 expref: -99 pid: 20469 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764784.244094:0:20469:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 3 00000080:00200000:1.0:1775764784.244101:0:20469:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764784.244107:0:20469:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764784.244112:0:20469:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cde00/0x36f9bdc9e421c7e6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb298 expref: -99 pid: 20469 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764784.244119:0:20469:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cde00/0x36f9bdc9e421c7e6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb298 expref: -99 pid: 20469 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764784.244125:0:20469:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764784.244128:0:20469:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764784.244130:0:20469:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c7e6 00000080:00010000:1.0:1775764784.244133:0:20469:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c7e6 from it ffffafbac19c7dc8 00010000:00010000:1.0:1775764784.244135:0:20469:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cde00/0x36f9bdc9e421c7e6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb298 expref: -99 pid: 20469 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764784.244141:0:20469:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cde00/0x36f9bdc9e421c7e6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb298 expref: -99 pid: 20469 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764784.244148:0:20469:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 3 00000080:00200000:1.0:1775764784.244153:0:20469:0:(namei.c:2216:do_mkdir()) VFS Op:name=d9.sanity-pfl, dir=[0x200000007:0x1:0x0](ffff8f9952d121c8) 00800000:00000004:1.0:1775764784.244177:0:20469:0:(obd_class.h:1058:obd_statfs_async()) lustre-MDT0000-mdc-ffff8f99523ee800: age 1381, max_age 0 00010000:00010000:1.0:1775764784.244234:0:20469:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cde00/0x36f9bdc9e421c7e6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fb298 expref: -99 pid: 20469 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764784.244271:0:20469:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cde00/0x36f9bdc9e421c7e6 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fb298 expref: -99 pid: 20469 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764784.244284:0:20469:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764784.244286:0:20469:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cde00/0x36f9bdc9e421c7e6 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fb298 expref: -99 pid: 20469 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1775764784.244313:0:20469:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1775764784.244320:0:20469:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e4700 x1862022967020288/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 560/4544 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764784.244340:0:20469:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job mkdir:3f83546c-01a9-4156-8a97-44c446245d3e:20469:1862022967020288:192.168.202.146@tcp:36:mkdir.0 00000100:00100000:1.0:1775764784.244431:0:20469:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f252600 going to sleep for 11 seconds 00000100:00100000:1.0:1775764784.247428:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1775764784.247458:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967020032:192.168.202.146@tcp:41:mkdir.0 00000100:00100000:0.0:1775764784.257921:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967020032:192.168.202.146@tcp:41:mkdir.0 00000100:00100000:2.0:1775764784.257921:0:20469:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job mkdir:3f83546c-01a9-4156-8a97-44c446245d3e:20469:1862022967020288:192.168.202.146@tcp:36:mkdir.0 00000080:00200000:2.0:1775764784.257970:0:20469:0:(llite_lib.c:3177:ll_read_inode2()) VFS Op:inode=[0x200000401:0xed8:0x0](ffff8f9952eb4b08) 00000080:00200000:2.0:1775764784.257978:0:20469:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0xed8:0x0], updating i_size 512 00000080:00200000:1.0:1775764784.266393:0:20471:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764784.266697:0:20471:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764784.266786:0:20471:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 20471 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.266797:0:20471:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 20471 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.266806:0:20471:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 20471 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764784.266833:0:20471:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5c00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:20471:1862022967020416:192.168.202.146@tcp:101:rm.0 00000100:00100000:1.0:1775764784.266939:0:20471:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f2529c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764784.283476:0:20471:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5c00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:20471:1862022967020416:192.168.202.146@tcp:101:rm.0 00010000:00010000:1.0:1775764784.283512:0:20471:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.283526:0:20471:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.283538:0:20471:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.283549:0:20471:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.283561:0:20471:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.283569:0:20471:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.283577:0:20471:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764784.283589:0:20471:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994f2e5c00 x1862022967020416/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764800 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764784.283603:0:20471:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.283617:0:20471:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1775764784.283623:0:20471:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1775764784.283649:0:20471:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 8704 00000080:00010000:1.0:1775764784.283656:0:20471:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c7ed 00000080:00010000:1.0:1775764784.283662:0:20471:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764784.283665:0:20471:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c7ed from it ffffafbac1ce3b58 00010000:00010000:1.0:1775764784.283669:0:20471:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.283678:0:20471:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764784.283716:0:20471:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764784.283782:0:20471:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=d9.sanity-pfl, dir=[0x200000007:0x1:0x0](ffff8f9952d121c8), flags=24 00000080:00200000:1.0:1775764784.283789:0:20471:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=d9.sanity-pfl, dir=[0x200000007:0x1:0x0](ffff8f9952d121c8), intent=0 00000002:00010000:1.0:1775764784.283811:0:20471:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: d9.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: lookup flags 0 00010000:00010000:1.0:1775764784.283863:0:20471:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c7f4 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 20471 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.283872:0:20471:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c7f4 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 20471 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.283882:0:20471:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c7f4 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 20471 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764784.283902:0:20471:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5c00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:20471:1862022967020544:192.168.202.146@tcp:101:rm.0 00000100:00100000:1.0:1775764784.284008:0:20471:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f2529c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764784.295343:0:20471:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5c00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:20471:1862022967020544:192.168.202.146@tcp:101:rm.0 00010000:00010000:1.0:1775764784.295380:0:20471:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c7f4 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2d0 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.295403:0:20471:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c7f4 lrc: 4/1,0 mode: --/PR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2d0 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.295420:0:20471:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c7f4 lrc: 4/1,0 mode: --/PR res: [0x200000401:0xed8:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2d0 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.295431:0:20471:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c7f4 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xed8:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb2d0 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.295442:0:20471:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c7f4 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xed8:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2d0 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.295454:0:20471:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c7f4 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xed8:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb2d0 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.295462:0:20471:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c7f4 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xed8:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb2d0 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.295469:0:20471:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c7f4 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xed8:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb2d0 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764784.295480:0:20471:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=b, status=0 req@ffff8f994f2e5c00 x1862022967020544/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/584 e 0 to 0 dl 1775764800 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764784.295492:0:20471:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c7f4 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xed8:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2d0 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.295511:0:20471:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/3800 (0 0) 00000080:00200000:1.0:1775764784.295517:0:20471:0:(namei.c:943:ll_intent_lock()) intent lock 16 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1775764784.295541:0:20471:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0xed8:0x0], updating i_size 512 00000080:00200000:1.0:1775764784.295545:0:20471:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0xed8:0x0](ffff8f9952eb4b08): rc = 0 00000080:00010000:1.0:1775764784.295550:0:20471:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xed8:0x0](ffff8f9952eb4b08) for lock 0x36f9bdc9e421c7f4 00000080:00010000:1.0:1775764784.295564:0:20471:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xed8:0x0](ffff8f9952eb4b08) 00000080:00010000:1.0:1775764784.295567:0:20471:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c7f4 from it ffffafbac1ce3a28 00010000:00010000:1.0:1775764784.295571:0:20471:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c7f4 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xed8:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb2d0 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.295578:0:20471:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c7f4 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xed8:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2d0 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764784.295628:0:20471:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xed8:0x0](ffff8f9952eb4b08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764784.295649:0:20471:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=f9.sanity-pfl, dir=[0x200000401:0xed8:0x0](ffff8f9952eb4b08), flags=8 00000080:00200000:1.0:1775764784.295654:0:20471:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=f9.sanity-pfl, dir=[0x200000401:0xed8:0x0](ffff8f9952eb4b08), intent=getattr 00000002:00010000:1.0:1775764784.295677:0:20471:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f9.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000401:0xed8:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764784.295721:0:20471:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421c7fb lrc: 3/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 20471 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.295728:0:20471:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421c7fb lrc: 3/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 20471 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.295766:0:20471:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421c7fb lrc: 3/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 20471 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764784.295788:0:20471:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5c00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:20471:1862022967020672:192.168.202.146@tcp:101:rm.0 00000100:00100000:1.0:1775764784.295894:0:20471:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f2529c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764784.301288:0:20471:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5c00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:20471:1862022967020672:192.168.202.146@tcp:101:rm.0 00010000:00010000:1.0:1775764784.301313:0:20471:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421c7fb lrc: 4/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 20471 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.301330:0:20471:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421c7fb lrc: 4/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 20471 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.301337:0:20471:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421c7fb lrc: 4/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 20471 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.301343:0:20471:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421c7fb lrc: 3/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 20471 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.301352:0:20471:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421c7fb lrc: 4/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 20471 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.301358:0:20471:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970641e00 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:1.0:1775764784.301362:0:20471:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421c7fb lrc: 4/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 20471 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.301382:0:20471:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421c7fb lrc: 4/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 20471 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.301390:0:20471:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421c7fb lrc: 5/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 20471 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.301397:0:20471:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1775764784.301403:0:20471:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421c7fb lrc: 3/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 20471 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.301417:0:20471:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421c7fb lrc: 0/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 20471 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764784.301434:0:20471:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=7, status=0 req@ffff8f994f2e5c00 x1862022967020672/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/584 e 0 to 0 dl 1775764800 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764784.301449:0:20471:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0xed8:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1775764784.301472:0:20471:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/3800 (0 0) 00000080:00200000:1.0:1775764784.301568:0:20471:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764784.301584:0:20471:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764784.301593:0:20471:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764784.301605:0:20471:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764784.301622:0:20471:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764784.301628:0:20471:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764784.301632:0:20471:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c7ed 00000080:00010000:1.0:1775764784.301636:0:20471:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c7ed from it ffffafbac1ce3c10 00010000:00010000:1.0:1775764784.301639:0:20471:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764784.301648:0:20471:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764784.301661:0:20471:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764784.301668:0:20471:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d9.sanity-pfl, flags=24 00000080:00200000:1.0:1775764784.301673:0:20471:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xed8:0x0](ffff8f9952eb4b08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764784.301685:0:20471:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=f9.sanity-pfl, dir=[0x200000401:0xed8:0x0](ffff8f9952eb4b08), flags=0 00000080:00200000:1.0:1775764784.301689:0:20471:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=f9.sanity-pfl, dir=[0x200000401:0xed8:0x0](ffff8f9952eb4b08), intent=getattr 00000002:00010000:1.0:1775764784.301701:0:20471:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f9.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000401:0xed8:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764784.301757:0:20471:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641000/0x36f9bdc9e421c802 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 20471 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.301765:0:20471:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641000/0x36f9bdc9e421c802 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 20471 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.301775:0:20471:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641000/0x36f9bdc9e421c802 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 20471 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764784.301796:0:20471:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5c00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:20471:1862022967020800:192.168.202.146@tcp:101:rm.0 00000100:00100000:1.0:1775764784.301896:0:20471:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f2529c0 going to sleep for 11 seconds 00000100:00100000:3.0:1775764784.326301:0:20471:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5c00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:20471:1862022967020800:192.168.202.146@tcp:101:rm.0 00010000:00010000:3.0:1775764784.326322:0:20471:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641000/0x36f9bdc9e421c802 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 20471 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764784.326336:0:20471:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641000/0x36f9bdc9e421c802 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 20471 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764784.326344:0:20471:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641000/0x36f9bdc9e421c802 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 20471 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764784.326350:0:20471:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641000/0x36f9bdc9e421c802 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 20471 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764784.326360:0:20471:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641000/0x36f9bdc9e421c802 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 20471 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764784.326366:0:20471:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970641000 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:3.0:1775764784.326371:0:20471:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641000/0x36f9bdc9e421c802 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 20471 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764784.326379:0:20471:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641000/0x36f9bdc9e421c802 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 20471 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764784.326385:0:20471:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641000/0x36f9bdc9e421c802 lrc: 5/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 20471 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764784.326392:0:20471:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1775764784.326400:0:20471:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641000/0x36f9bdc9e421c802 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 20471 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764784.326406:0:20471:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641000/0x36f9bdc9e421c802 lrc: 0/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 20471 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764784.326422:0:20471:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=7, status=0 req@ffff8f994f2e5c00 x1862022967020800/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/584 e 0 to 0 dl 1775764800 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1775764784.326440:0:20471:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0xed8:0x0] suppgids 0 -1: rc 0 00010000:00010000:3.0:1775764784.326466:0:20471:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/3800 (0 0) 00000080:00200000:2.0:1775764784.339601:0:20475:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:2.0:1775764784.339632:0:20475:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764784.339652:0:20475:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764784.339669:0:20475:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764784.339678:0:20475:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764784.339686:0:20475:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:2.0:1775764784.339690:0:20475:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c7ed 00000080:00010000:2.0:1775764784.339697:0:20475:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c7ed from it ffffafbac0dd3b58 00010000:00010000:2.0:1775764784.339701:0:20475:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764784.339709:0:20475:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764784.339732:0:20475:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:2.0:1775764784.339823:0:20475:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d9.sanity-pfl, flags=24 00000080:00200000:2.0:1775764784.339830:0:20475:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xed8:0x0](ffff8f9952eb4b08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764784.339842:0:20475:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=f9.sanity-pfl, dir=[0x200000401:0xed8:0x0](ffff8f9952eb4b08), flags=8 00000080:00200000:2.0:1775764784.339846:0:20475:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=f9.sanity-pfl, dir=[0x200000401:0xed8:0x0](ffff8f9952eb4b08), intent=getattr 00000002:00010000:2.0:1775764784.339858:0:20475:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f9.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000401:0xed8:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764784.339903:0:20475:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebe00/0x36f9bdc9e421c809 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 20475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764784.339910:0:20475:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebe00/0x36f9bdc9e421c809 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 20475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764784.339920:0:20475:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebe00/0x36f9bdc9e421c809 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 20475 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764784.339950:0:20475:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:20475:1862022967020928:192.168.202.146@tcp:101:lfs.0 00000100:00100000:2.0:1775764784.340065:0:20475:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560b40 going to sleep for 11 seconds 00000100:00100000:3.0:1775764784.347146:0:20475:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:20475:1862022967020928:192.168.202.146@tcp:101:lfs.0 00010000:00010000:3.0:1775764784.347169:0:20475:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebe00/0x36f9bdc9e421c809 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 20475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764784.347182:0:20475:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebe00/0x36f9bdc9e421c809 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 20475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764784.347189:0:20475:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebe00/0x36f9bdc9e421c809 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 20475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764784.347196:0:20475:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebe00/0x36f9bdc9e421c809 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 20475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764784.347204:0:20475:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebe00/0x36f9bdc9e421c809 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 20475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764784.347210:0:20475:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99706ebe00 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:3.0:1775764784.347214:0:20475:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebe00/0x36f9bdc9e421c809 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 20475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764784.347225:0:20475:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebe00/0x36f9bdc9e421c809 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 20475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764784.347232:0:20475:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebe00/0x36f9bdc9e421c809 lrc: 5/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 20475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764784.347240:0:20475:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1775764784.347247:0:20475:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebe00/0x36f9bdc9e421c809 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 20475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764784.347252:0:20475:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebe00/0x36f9bdc9e421c809 lrc: 0/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 20475 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764784.347286:0:20475:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=7, status=0 req@ffff8f99703d0380 x1862022967020928/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/584 e 0 to 0 dl 1775764800 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1775764784.347303:0:20475:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0xed8:0x0] suppgids 0 -1: rc 0 00010000:00010000:3.0:1775764784.347338:0:20475:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/3800 (0 0) 00000080:00200000:3.0:1775764784.347424:0:20475:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:3.0:1775764784.347441:0:20475:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764784.347450:0:20475:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764784.347461:0:20475:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764784.347468:0:20475:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764784.347472:0:20475:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:3.0:1775764784.347475:0:20475:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c7ed 00000080:00010000:3.0:1775764784.347479:0:20475:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c7ed from it ffffafbac0dd3bf8 00010000:00010000:3.0:1775764784.347483:0:20475:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764784.347491:0:20475:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1775764784.347517:0:20475:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:3.0:1775764784.347525:0:20475:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d9.sanity-pfl, flags=25 00000080:00200000:3.0:1775764784.347529:0:20475:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xed8:0x0](ffff8f9952eb4b08), inode mode 41ed mask 1 00000080:00200000:3.0:1775764784.347542:0:20475:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xed8:0x0](ffff8f9952eb4b08), inode mode 41ed mask 3 00000080:00200000:3.0:1775764784.347547:0:20475:0:(namei.c:1399:ll_atomic_open()) VFS Op:name=f9.sanity-pfl, dir=[0x200000401:0xed8:0x0](ffff8f9952eb4b08), file ffff8f994eeb4c00, open_flags a141, mode 81b6 opened 0 00000080:00200000:3.0:1775764784.347555:0:20475:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=f9.sanity-pfl, dir=[0x200000401:0xed8:0x0](ffff8f9952eb4b08), intent=open|creat 00000002:00010000:3.0:1775764784.347576:0:20475:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f9.sanity-pfl,[0x200000401:0xed9:0x0]) in obj [0x200000401:0xed8:0x0], intent: open|creat flags 0120502 00010000:00010000:3.0:1775764784.347630:0:20475:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4a00/0x36f9bdc9e421c810 lrc: 3/0,1 mode: --/CW res: [0x200000401:0xed8:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 20475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764784.347637:0:20475:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4a00/0x36f9bdc9e421c810 lrc: 3/0,1 mode: --/CW res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 20475 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:3.0:1775764784.347650:0:20475:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1775764784.347657:0:20475:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970edd180 x1862022967021184/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 528/70288 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1775764784.347667:0:20475:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4a00/0x36f9bdc9e421c810 lrc: 3/0,1 mode: --/CW res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 20475 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764784.347689:0:20475:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:20475:1862022967021184:192.168.202.146@tcp:101:lfs.0 00000100:00100000:3.0:1775764784.347878:0:20475:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9951fa9240 going to sleep for 11 seconds 00000100:00100000:3.0:1775764784.368441:0:20475:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:20475:1862022967021184:192.168.202.146@tcp:101:lfs.0 00010000:00010000:3.0:1775764784.368468:0:20475:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4a00/0x36f9bdc9e421c810 lrc: 4/0,1 mode: --/CW res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb32b expref: -99 pid: 20475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764784.368490:0:20475:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4a00/0x36f9bdc9e421c810 lrc: 4/0,1 mode: --/PR res: [0x200000401:0xed9:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb32b expref: -99 pid: 20475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764784.368507:0:20475:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4a00/0x36f9bdc9e421c810 lrc: 4/0,1 mode: --/PR res: [0x200000401:0xed9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb32b expref: -99 pid: 20475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764784.368519:0:20475:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4a00/0x36f9bdc9e421c810 lrc: 4/0,1 mode: PR/PR res: [0x200000401:0xed9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb32b expref: -99 pid: 20475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764784.368530:0:20475:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4a00/0x36f9bdc9e421c810 lrc: 4/0,1 mode: PR/PR res: [0x200000401:0xed9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb32b expref: -99 pid: 20475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764784.368543:0:20475:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4a00/0x36f9bdc9e421c810 lrc: 5/1,1 mode: PR/PR res: [0x200000401:0xed9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb32b expref: -99 pid: 20475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764784.368551:0:20475:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4a00/0x36f9bdc9e421c810 lrc: 5/1,1 mode: PR/PR res: [0x200000401:0xed9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb32b expref: -99 pid: 20475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764784.368558:0:20475:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4a00/0x36f9bdc9e421c810 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xed9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb32b expref: -99 pid: 20475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764784.368572:0:20475:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=3 disposition=8000037, status=0 req@ffff8f9970edd180 x1862022967021184/t4294997598(4294997598) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 528/608 e 0 to 0 dl 1775764800 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:3.0:1775764784.368586:0:20475:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f9970edd180 x1862022967021184/t4294997598(4294997598) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 528/608 e 0 to 0 dl 1775764800 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1775764784.368598:0:20475:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4a00/0x36f9bdc9e421c810 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xed9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb32b expref: -99 pid: 20475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764784.368609:0:20475:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/3801 (0 0) 00000080:00200000:3.0:1775764784.368628:0:20475:0:(namei.c:943:ll_intent_lock()) intent lock 3 on i1 [0x200000401:0xed8:0x0] suppgids 0 -1: rc 0 00000080:00200000:3.0:1775764784.368648:0:20475:0:(namei.c:1610:ll_create_it()) VFS Op:name=f9.sanity-pfl, dir=[0x200000401:0xed8:0x0](ffff8f9952eb4b08), intent=open|creat 00000080:00200000:3.0:1775764784.368684:0:20475:0:(llite_lib.c:3177:ll_read_inode2()) VFS Op:inode=[0x200000401:0xed9:0x0](ffff8f9952ee9148) 00000080:00200000:3.0:1775764784.368692:0:20475:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0xed9:0x0], updating i_size 0 00000080:00200000:3.0:1775764784.371774:0:20475:0:(file.c:6766:ll_layout_conf()) [0x200000401:0xed9:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:3.0:1775764784.371819:0:20475:0:(namei.c:1576:ll_create_node()) setting l_ast_data to inode [0x200000401:0xed9:0x0](ffff8f9952ee9148) 00000080:00010000:3.0:1775764784.371823:0:20475:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xed9:0x0](ffff8f9952ee9148) for lock 0x36f9bdc9e421c810 00000080:00010000:3.0:1775764784.371838:0:20475:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xed9:0x0](ffff8f9952ee9148) 00000080:00200008:3.0:1775764784.371850:0:20475:0:(file.c:1057:ll_file_open()) START file f9.sanity-pfl:[0x200000401:0xed9:0x0](ffff8f9952ee9148), flags 120501 00000002:00100000:3.0:1775764784.371879:0:20475:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f9970edd180 x1862022967021184/t4294997598(4294997598) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 528/608 e 0 to 0 dl 1775764800 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:3.0:1775764784.371920:0:20475:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c810 from it ffff8f99704b75a0 00010000:00010000:3.0:1775764784.371927:0:20475:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4a00/0x36f9bdc9e421c810 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xed9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fb32b expref: -99 pid: 20475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764784.371939:0:20475:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4a00/0x36f9bdc9e421c810 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xed9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fb32b expref: -99 pid: 20475 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764784.371961:0:20475:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xed9:0x0](ffff8f9952ee9148), inode mode 81a4 mask 40 00000080:00200000:3.0:1775764784.372004:0:20475:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xed9:0x0](ffff8f9952ee9148), inode mode 81a4 mask 2 00000080:00200000:3.0:1775764784.372013:0:20475:0:(xattr.c:410:ll_xattr_set()) VFS Op:inode=[0x200000401:0xed9:0x0](ffff8f9952ee9148), xattr lov 00000002:00010000:3.0:1775764784.372030:0:20475:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xed9:0x0]) in obj [0x200000401:0xed8:0x0], intent: open flags 040000002 00010000:00010000:3.0:1775764784.372086:0:20475:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c817 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 20475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764784.372094:0:20475:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c817 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 20475 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:3.0:1775764784.372106:0:20475:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1775764784.372113:0:20475:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970edd880 x1862022967021440/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/135824 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1775764784.372122:0:20475:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c817 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 20475 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764784.372144:0:20475:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:20475:1862022967021440:192.168.202.146@tcp:101:lfs.0 00000100:00100000:3.0:1775764784.372335:0:20475:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9951fa9540 going to sleep for 11 seconds 00000100:00100000:0.0:1775764784.388864:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764784.388907:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978399872 00000100:00100000:0.0:1775764784.388917:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978399872 00000100:00100000:0.0:1775764784.388977:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24224 00000100:00100000:0.0:1775764784.389186:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970a76a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+5:5795:x1862022978399872:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764784.389206:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4a00/0x36f9bdc9e421c810 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xed9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x2aac6a50e5fb32b expref: -99 pid: 20475 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764784.389293:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764784.389301:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970a76a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+5:5795:x1862022978399872:12345-192.168.202.146@tcp:104: Request processed in 217us (445us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764784.389309:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24224 00010000:00010000:0.0:1775764784.389336:0:19800:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1775764784.389342:0:19800:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4a00/0x36f9bdc9e421c810 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xed9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x2aac6a50e5fb32b expref: -99 pid: 20475 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764784.389353:0:19800:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99435d4a00 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:0.0:1775764784.389358:0:19800:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4a00/0x36f9bdc9e421c810 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xed9:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x2aac6a50e5fb32b expref: -99 pid: 20475 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764784.389367:0:19800:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4a00/0x36f9bdc9e421c810 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xed9:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x2aac6a50e5fb32b expref: -99 pid: 20475 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764784.389605:0:19800:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4a00/0x36f9bdc9e421c810 lrc: 3/0,0 mode: PR/PR res: [0x200000401:0xed9:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x2aac6a50e5fb32b expref: -99 pid: 20475 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764784.389667:0:19800:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0xed9:0x0]: losing layout lock 00010000:00010000:0.0:1775764784.389756:0:19800:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4a00/0x36f9bdc9e421c810 lrc: 2/0,0 mode: --/PR res: [0x200000401:0xed9:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x2aac6a50e5fb32b expref: -99 pid: 20475 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764784.389763:0:19800:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764784.389788:0:19800:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4a00/0x36f9bdc9e421c810 lrc: 1/0,0 mode: --/PR res: [0x200000401:0xed9:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x2aac6a50e5fb32b expref: -99 pid: 20475 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764784.389796:0:19800:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4a00/0x36f9bdc9e421c810 lrc: 0/0,0 mode: --/PR res: [0x200000401:0xed9:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x2aac6a50e5fb32b expref: -99 pid: 20475 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764784.389814:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764784.389817:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764784.389819:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764784.389836:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967021568:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764784.412898:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967021568:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764784.415480:0:20475:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:20475:1862022967021440:192.168.202.146@tcp:101:lfs.0 00010000:00010000:3.0:1775764784.415516:0:20475:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c817 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 20475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764784.415529:0:20475:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c817 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 20475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764784.415537:0:20475:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c817 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 20475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764784.415543:0:20475:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c817 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 20475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764784.415551:0:20475:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c817 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 20475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764784.415560:0:20475:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99435d5400 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:3.0:1775764784.415565:0:20475:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c817 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 20475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764784.415572:0:20475:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c817 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 20475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764784.415579:0:20475:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c817 lrc: 5/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 20475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764784.415586:0:20475:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1775764784.415592:0:20475:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c817 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 20475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764784.415598:0:20475:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c817 lrc: 0/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 20475 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764784.415616:0:20475:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=800002b, status=0 req@ffff8f9970edd880 x1862022967021440/t4294997599(4294997599) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/848 e 0 to 0 dl 1775764800 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:3.0:1775764784.415629:0:20475:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f9970edd880 x1862022967021440/t4294997599(4294997599) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/848 e 0 to 0 dl 1775764800 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1775764784.415649:0:20475:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0xed8:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1775764784.415770:0:20475:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xed9:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764784.415779:0:20475:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xed9:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764784.415848:0:20475:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0xed9:0x0], updating i_size 0 00000002:00100000:3.0:1775764784.415865:0:20475:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f9970edd880 x1862022967021440/t4294997599(4294997599) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 760/848 e 0 to 0 dl 1775764800 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:3.0:1775764784.415907:0:20475:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f9970edd880 x1862022967021440/t4294997599(4294997599) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 760/848 e 0 to 0 dl 1775764800 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:3.0:1775764784.415917:0:20475:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:3.0:1775764784.415921:0:20475:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970edc380 x1862022967021824/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:3.0:1775764784.415942:0:20475:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:20475:1862022967021824:192.168.202.146@tcp:35:lfs.0 00000100:00100000:3.0:1775764784.416033:0:20475:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9951fa9540 going to sleep for 11 seconds 00000100:00100000:1.0:1775764784.429290:0:20475:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:20475:1862022967021824:192.168.202.146@tcp:35:lfs.0 00000002:00100000:1.0:1775764784.429331:0:20475:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f9970edd880 x1862022967021440/t4294997599(4294997599) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 760/848 e 0 to 0 dl 1775764800 ref 2 fl Complete:RQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200008:1.0:1775764784.429403:0:20475:0:(file.c:447:ll_file_release()) START file f9.sanity-pfl:[0x200000401:0xed9:0x0](ffff8f9952ee9148), flags 100001 00000002:00100000:1.0:1775764784.429466:0:20475:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f9970edd180 x1862022967021184/t4294997598(4294997598) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 528/608 e 0 to 0 dl 1775764800 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1775764784.429477:0:20475:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764784.429481:0:20475:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e4380 x1862022967022080/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764784.429515:0:20475:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4380 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:20475:1862022967022080:192.168.202.146@tcp:35:lfs.0 00000100:00100000:1.0:1775764784.429613:0:20475:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f252540 going to sleep for 11 seconds 00000100:00100000:3.0:1775764784.434434:0:20475:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4380 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:20475:1862022967022080:192.168.202.146@tcp:35:lfs.0 00000002:00100000:3.0:1775764784.434469:0:20475:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f9970edd180 x1862022967021184/t4294997598(4294997598) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 528/608 e 0 to 0 dl 1775764800 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764784.460365:0:20478:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764784.460412:0:20478:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764784.460430:0:20478:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764784.460446:0:20478:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764784.460455:0:20478:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764784.460463:0:20478:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764784.460467:0:20478:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c7ed 00000080:00010000:1.0:1775764784.460474:0:20478:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c7ed from it ffffafbac0c87bf8 00010000:00010000:1.0:1775764784.460478:0:20478:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764784.460486:0:20478:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764784.460511:0:20478:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764784.460520:0:20478:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d9.sanity-pfl, flags=27 00000080:00200000:1.0:1775764784.460526:0:20478:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xed8:0x0](ffff8f9952eb4b08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764784.460539:0:20478:0:(namei.c:1399:ll_atomic_open()) VFS Op:name=f9.sanity-pfl, dir=[0x200000401:0xed8:0x0](ffff8f9952eb4b08), file ffff8f9961382a80, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1775764784.460547:0:20478:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=f9.sanity-pfl, dir=[0x200000401:0xed8:0x0](ffff8f9952eb4b08), intent=open 00000002:00010000:1.0:1775764784.460563:0:20478:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f9.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000401:0xed8:0x0], intent: open flags 0304001 00010000:00010000:1.0:1775764784.460628:0:20478:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c81e lrc: 3/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 20478 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764784.460636:0:20478:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c81e lrc: 3/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 20478 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764784.460649:0:20478:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764784.460664:0:20478:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e6680 x1862022967022336/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4624/135824 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764784.460674:0:20478:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c81e lrc: 3/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 20478 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764784.460702:0:20478:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:20478:1862022967022336:192.168.202.146@tcp:101:lfs.0 00000100:00100000:1.0:1775764784.460930:0:20478:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f252780 going to sleep for 11 seconds 00000100:00100000:2.0:1775764784.466710:0:20478:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:20478:1862022967022336:192.168.202.146@tcp:101:lfs.0 00010000:00010000:2.0:1775764784.466735:0:20478:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c81e lrc: 4/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 20478 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764784.466783:0:20478:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c81e lrc: 4/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 20478 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764784.466790:0:20478:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c81e lrc: 4/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 20478 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764784.466797:0:20478:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c81e lrc: 3/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 20478 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764784.466807:0:20478:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c81e lrc: 4/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 20478 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764784.466813:0:20478:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970640a00 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:2.0:1775764784.466818:0:20478:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c81e lrc: 4/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 20478 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764784.466824:0:20478:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c81e lrc: 4/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 20478 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764784.466831:0:20478:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c81e lrc: 5/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 20478 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764784.466839:0:20478:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1775764784.466845:0:20478:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c81e lrc: 3/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 20478 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764784.466851:0:20478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640a00/0x36f9bdc9e421c81e lrc: 0/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 20478 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764784.466867:0:20478:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff8f994f2e6680 x1862022967022336/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4624/608 e 0 to 0 dl 1775764800 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764784.466880:0:20478:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0xed8:0x0] suppgids 0 -1: rc -20 00000080:00200000:2.0:1775764784.467035:0:20478:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:2.0:1775764784.467056:0:20478:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764784.467073:0:20478:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764784.467295:0:20478:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1775764784.467306:0:20478:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764784.467313:0:20478:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:2.0:1775764784.467317:0:20478:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c7ed 00000080:00010000:2.0:1775764784.467324:0:20478:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c7ed from it ffffafbac0c87bf8 00010000:00010000:2.0:1775764784.467327:0:20478:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764784.467336:0:20478:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:2.0:1775764784.467352:0:20478:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:2.0:1775764784.467360:0:20478:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d9.sanity-pfl, flags=267 00000080:00200000:2.0:1775764784.467367:0:20478:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xed8:0x0](ffff8f9952eb4b08), inode mode 41ed mask 44 00000080:00200008:2.0:1775764784.467453:0:20478:0:(file.c:1057:ll_file_open()) START file d9.sanity-pfl:[0x200000401:0xed8:0x0](ffff8f9952eb4b08), flags 304000 00000002:00010000:2.0:1775764784.467470:0:20478:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xed8:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100240304001 00010000:00010000:2.0:1775764784.467533:0:20478:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb400/0x36f9bdc9e421c825 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 20478 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764784.467541:0:20478:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb400/0x36f9bdc9e421c825 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 20478 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764784.467551:0:20478:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764784.467559:0:20478:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f99703d2680 x1862022967022592/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/135824 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764784.467568:0:20478:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb400/0x36f9bdc9e421c825 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 20478 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764784.467594:0:20478:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:20478:1862022967022592:192.168.202.146@tcp:101:lfs.0 00000100:00100000:2.0:1775764784.468753:0:20478:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560a80 going to sleep for 11 seconds 00000100:00100000:2.0:1775764784.489312:0:20478:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:20478:1862022967022592:192.168.202.146@tcp:101:lfs.0 00010000:00010000:2.0:1775764784.489339:0:20478:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb400/0x36f9bdc9e421c825 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 20478 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764784.489354:0:20478:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb400/0x36f9bdc9e421c825 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 20478 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764784.489362:0:20478:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb400/0x36f9bdc9e421c825 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 20478 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764784.489385:0:20478:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb400/0x36f9bdc9e421c825 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 20478 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764784.489395:0:20478:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb400/0x36f9bdc9e421c825 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 20478 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764784.489402:0:20478:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99706eb400 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:2.0:1775764784.489407:0:20478:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb400/0x36f9bdc9e421c825 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 20478 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764784.489414:0:20478:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb400/0x36f9bdc9e421c825 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 20478 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764784.489421:0:20478:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb400/0x36f9bdc9e421c825 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 20478 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764784.489429:0:20478:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1775764784.489435:0:20478:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb400/0x36f9bdc9e421c825 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 20478 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764784.489442:0:20478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb400/0x36f9bdc9e421c825 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 20478 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764784.489461:0:20478:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8f99703d2680 x1862022967022592/t4294997602(4294997602) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764800 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764784.489475:0:20478:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f99703d2680 x1862022967022592/t4294997602(4294997602) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764800 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764784.489488:0:20478:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764784.489534:0:20478:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0xed8:0x0], updating i_size 512 00000002:00100000:2.0:1775764784.489554:0:20478:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f99703d2680 x1862022967022592/t4294997602(4294997602) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764800 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:2.0:1775764784.489661:0:20478:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000401:0xed8:0x0](ffff8f9952eb4b08) cmd=c0086915 arg=9463d0 00000100:00100000:2.0:1775764784.489728:0:20478:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:20478:1862022967022720:192.168.202.146@tcp:34:lfs.0 00000100:00100000:2.0:1775764784.490179:0:20478:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560a80 going to sleep for 11 seconds 00000100:00100000:2.0:1775764784.493579:0:20478:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:20478:1862022967022720:192.168.202.146@tcp:34:lfs.0 00000080:00200008:2.0:1775764784.493788:0:20478:0:(file.c:447:ll_file_release()) START file d9.sanity-pfl:[0x200000401:0xed8:0x0](ffff8f9952eb4b08), flags 304000 00000002:00100000:2.0:1775764784.493863:0:20478:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f99703d2680 x1862022967022592/t4294997602(4294997602) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764800 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764784.493877:0:20478:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764784.493882:0:20478:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f99703d1500 x1862022967022976/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764784.493904:0:20478:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:20478:1862022967022976:192.168.202.146@tcp:35:lfs.0 00000100:00100000:2.0:1775764784.493989:0:20478:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560a80 going to sleep for 11 seconds 00000100:00100000:2.0:1775764784.500591:0:20478:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:20478:1862022967022976:192.168.202.146@tcp:35:lfs.0 00000002:00100000:2.0:1775764784.500627:0:20478:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f99703d2680 x1862022967022592/t4294997602(4294997602) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764800 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00010000:00010000:0.0:1775764784.650355:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764784.838135:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:884ae1b0-72f8-4a88-a129-db99cded7efc:2381:1862022967023104:192.168.202.146@tcp:400:kworker.0 00000100:00100000:2.0:1775764784.838250:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967023360:192.168.202.146@tcp:400:kworker.0 00000100:00100000:2.0:1775764784.838299:0:2381:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1775764784.838306:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967023232:192.168.202.146@tcp:400:kworker.0 00000100:00100000:2.0:1775764784.853772:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967023232:192.168.202.146@tcp:400:kworker.0 00000100:00100000:2.0:1775764784.853788:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:884ae1b0-72f8-4a88-a129-db99cded7efc:2381:1862022967023104:192.168.202.146@tcp:400:kworker.0 00000100:00100000:2.0:1775764784.853799:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967023360:192.168.202.146@tcp:400:kworker.0 00010000:00010000:0.0:1775764785.670327:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:1.0:1775764785.781248:0:20488:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8f9952d121c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764785.781263:0:20488:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764785.781284:0:20488:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764785.781305:0:20488:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764785.781320:0:20488:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:1.0:1775764785.781327:0:20488:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764785.781334:0:20488:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764785.781338:0:20488:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c7ed 00000080:00010000:1.0:1775764785.781345:0:20488:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c7ed from it ffffafbac1c93cb8 00010000:00010000:1.0:1775764785.781349:0:20488:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764785.781358:0:20488:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200008:1.0:1775764785.781995:0:20488:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8f9952d121c8), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1775764785.782001:0:20488:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764785.782012:0:20488:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764785.782023:0:20488:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764785.782039:0:20488:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1775764785.782045:0:20488:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764785.782050:0:20488:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764785.782053:0:20488:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c7ed 00000080:00010000:1.0:1775764785.782058:0:20488:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c7ed from it ffffafbac1c93cb8 00010000:00010000:1.0:1775764785.782061:0:20488:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764785.782069:0:20488:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1775764785.782156:0:20488:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764785.782164:0:20488:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764785.782171:0:20488:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764785.782180:0:20488:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1775764785.782186:0:20488:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764785.782190:0:20488:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764785.782192:0:20488:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c7ed 00000080:00010000:1.0:1775764785.782195:0:20488:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c7ed from it ffffafbac1c93bb0 00010000:00010000:1.0:1775764785.782198:0:20488:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764785.782205:0:20488:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:1.0:1775764785.782213:0:20488:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 44 00000080:00200008:1.0:1775764785.782221:0:20488:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8f9952d121c8), flags 100000 00000080:00200080:1.0:1775764785.782296:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:1.0:1775764785.782314:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9970e09c00 uarg=0000000000000000 00800000:00000004:1.0:1775764785.782320:0:20488:0:(obd_class.h:1092:obd_statfs()) lustre-MDT0000-mdc-ffff8f99523ee800: age 1381, max_age 1411 00000100:00100000:1.0:1775764785.782371:0:20488:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7100 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:20488:1862022967023488:192.168.202.146@tcp:41:lfs.0 00000100:00100000:1.0:1775764785.782476:0:20488:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f252780 going to sleep for 11 seconds 00000100:00100000:3.0:1775764785.789438:0:20488:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7100 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:20488:1862022967023488:192.168.202.146@tcp:41:lfs.0 00800000:00000004:3.0:1775764785.789463:0:20488:0:(obd_class.h:1118:obd_statfs()) lustre-MDT0000-mdc-ffff8f99523ee800: update ffff8f9952360450 cache blocks 17232/17269 objects 517979/518328 00000080:00200080:3.0:1775764785.789559:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789575:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789582:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789586:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789589:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789592:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789596:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789599:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789602:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789605:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789609:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789612:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789616:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789618:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789622:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789625:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789628:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789631:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789634:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789637:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789641:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789643:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789647:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789650:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789653:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789655:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789659:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789661:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789664:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789667:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789670:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789673:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789676:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789679:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789682:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789686:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789690:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789693:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789696:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789699:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789703:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789706:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789710:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789713:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789716:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789719:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789722:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789725:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789728:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789731:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789735:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789785:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789789:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789792:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789795:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789798:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789801:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789812:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789817:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789820:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789824:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789827:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789831:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789834:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789838:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789841:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789844:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789846:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789850:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789852:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789856:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789859:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789862:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789864:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789868:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789870:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789874:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789876:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789880:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789882:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789885:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789888:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789891:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789894:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789897:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789900:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789903:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789906:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789909:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789912:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789914:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789917:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789920:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789923:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789926:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789929:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789932:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789935:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789938:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789941:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789945:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789948:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789951:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789954:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:3.0:1775764785.789958:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:3.0:1775764785.789961:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966930c00 uarg=0000000000000000 00000080:00200080:2.0:1775764785.790434:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:2.0:1775764785.790444:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.790449:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:2.0:1775764785.790452:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.790455:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:2.0:1775764785.790459:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.790462:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:2.0:1775764785.790465:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.790468:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:2.0:1775764785.790470:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.790475:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:2.0:1775764785.790491:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.790495:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:2.0:1775764785.790499:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.790503:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:2.0:1775764785.790506:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.790510:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:2.0:1775764785.790513:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.790516:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:2.0:1775764785.790519:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.790523:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00800000:00000080:2.0:1775764785.790526:0:20488:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.790530:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.790536:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00020000:00000004:2.0:1775764785.790544:0:20488:0:(obd_class.h:1092:obd_statfs()) lustre-OST0000-osc-ffff8f99523ee800: age 257, max_age 1411 00000100:00100000:2.0:1775764785.790590:0:20488:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:20488:1862022967023616:192.168.202.146@tcp:13:lfs.0 00000100:00100000:2.0:1775764785.790679:0:20488:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560a80 going to sleep for 11 seconds 00000100:00100000:2.0:1775764785.825650:0:20488:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:20488:1862022967023616:192.168.202.146@tcp:13:lfs.0 00020000:00000004:2.0:1775764785.825693:0:20488:0:(obd_class.h:1118:obd_statfs()) lustre-OST0000-osc-ffff8f99523ee800: update ffff8f99523637d0 cache blocks 3678/3683 objects 117760/119384 00000080:00200080:2.0:1775764785.825983:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.826002:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00020000:00000004:2.0:1775764785.826011:0:20488:0:(obd_class.h:1092:obd_statfs()) lustre-OST0001-osc-ffff8f99523ee800: age 257, max_age 1411 00000100:00100000:2.0:1775764785.826052:0:20488:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:20488:1862022967023744:192.168.202.146@tcp:13:lfs.0 00000100:00100000:2.0:1775764785.826152:0:20488:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560a80 going to sleep for 11 seconds 00000100:00100000:2.0:1775764785.843818:0:20488:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:20488:1862022967023744:192.168.202.146@tcp:13:lfs.0 00020000:00000004:2.0:1775764785.843847:0:20488:0:(obd_class.h:1118:obd_statfs()) lustre-OST0001-osc-ffff8f99523ee800: update ffff8f9951ec8450 cache blocks 3655/3683 objects 117760/119384 00000080:00200080:2.0:1775764785.843982:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844001:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844009:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844013:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844018:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844021:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844026:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844030:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844034:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844038:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844042:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844046:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844049:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844053:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844057:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844061:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844065:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844069:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844073:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844076:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844079:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844082:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844086:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844090:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844094:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844097:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844100:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844103:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844106:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844110:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844113:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844117:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844121:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844124:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844127:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844130:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844134:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844138:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844141:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844145:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844149:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844152:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844156:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844160:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844164:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844168:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844172:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844175:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844180:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844183:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844186:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844189:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844193:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844196:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844199:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844202:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844205:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844282:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844289:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844294:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844298:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844302:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844305:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844309:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844313:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844316:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844319:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844323:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844327:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844330:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844333:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844337:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844340:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844344:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844347:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844351:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844354:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844357:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844361:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844364:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844380:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844385:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844389:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844392:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844396:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844399:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844403:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844406:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844410:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844413:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844417:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844421:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844425:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844428:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844432:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844436:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844439:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844443:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844447:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844451:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844455:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844459:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844462:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844465:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844469:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844472:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844477:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844493:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844498:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844501:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844505:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844508:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844512:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844515:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844519:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844522:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844525:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844529:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844533:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844536:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844540:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844543:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844546:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844549:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200080:2.0:1775764785.844554:0:20488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fff4d37b9f0 00020000:00000080:2.0:1775764785.844556:0:20488:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f994f0cf800 uarg=0000000000000000 00000080:00200008:2.0:1775764785.844572:0:20488:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8f9952d121c8), flags 100000 00000080:00200000:1.0:1775764785.972097:0:20495:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764785.972127:0:20495:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764785.972147:0:20495:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:1.0:1775764785.972164:0:20495:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:1.0:1775764785.972173:0:20495:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764785.972181:0:20495:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764785.972185:0:20495:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c7ed 00000080:00010000:1.0:1775764785.972193:0:20495:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c7ed from it ffffafbac1463bd0 00010000:00010000:1.0:1775764785.972197:0:20495:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:1.0:1775764785.972207:0:20495:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:1.0:1775764785.972225:0:20495:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764785.972240:0:20495:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=fsa-oleg246-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8f9952d121c8), flags=1536 00000080:00200000:1.0:1775764785.972244:0:20495:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764785.972250:0:20495:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764785.972256:0:20495:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764785.972264:0:20495:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:1.0:1775764785.972271:0:20495:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764785.972274:0:20495:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764785.972277:0:20495:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c7ed 00000080:00010000:1.0:1775764785.972281:0:20495:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c7ed from it ffffafbac1463cd0 00010000:00010000:1.0:1775764785.972283:0:20495:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764785.972291:0:20495:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00000080:00200000:1.0:1775764785.972298:0:20495:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 3 00000080:00200000:1.0:1775764785.972305:0:20495:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764785.972310:0:20495:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764785.972316:0:20495:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00010000:00010000:1.0:1775764785.972329:0:20495:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00000080:00200000:1.0:1775764785.972335:0:20495:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764785.972338:0:20495:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764785.972341:0:20495:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c7ed 00000080:00010000:1.0:1775764785.972344:0:20495:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c7ed from it ffffafbac1463db0 00010000:00010000:1.0:1775764785.972346:0:20495:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00010000:00010000:1.0:1775764785.972353:0:20495:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 0 00000080:00200000:1.0:1775764785.972361:0:20495:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 3 00000080:00200000:1.0:1775764785.972366:0:20495:0:(namei.c:2106:ll_create_nd()) VFS Op:name=fsa-oleg246-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8f9952d121c8), flags=33188, excl=1 00000080:00200000:1.0:1775764785.972369:0:20495:0:(namei.c:2061:ll_mknod()) VFS Op:name=fsa-oleg246-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8f9952d121c8) mode 100644 dev 0 00010000:00010000:1.0:1775764785.972396:0:20495:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 0 00010000:00010000:1.0:1775764785.972450:0:20495:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 0 00010000:00010000:1.0:1775764785.972458:0:20495:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764785.972461:0:20495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c7ed lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fb2b4 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 0 00000020:00100000:1.0:1775764785.972499:0:20495:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1775764785.972514:0:20495:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e7480 x1862022967024000/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 528/4544 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764785.972539:0:20495:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7480 pname:cluuid:pid:xid:nid:opc:job mcreate:3f83546c-01a9-4156-8a97-44c446245d3e:20495:1862022967024000:192.168.202.146@tcp:36:mcreate.0 00000100:00100000:1.0:1775764785.972650:0:20495:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f252300 going to sleep for 11 seconds 00000100:00100000:2.0:1775764785.981253:0:20495:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7480 pname:cluuid:pid:xid:nid:opc:job mcreate:3f83546c-01a9-4156-8a97-44c446245d3e:20495:1862022967024000:192.168.202.146@tcp:36:mcreate.0 00000080:00200000:2.0:1775764785.981307:0:20495:0:(llite_lib.c:3177:ll_read_inode2()) VFS Op:inode=[0x200000401:0xeda:0x0](ffff8f9952eb5b88) 00000080:00200000:2.0:1775764785.981388:0:20495:0:(namei.c:2114:ll_create_nd()) VFS Op:name=fsa-oleg246-client.virtnet, unhashed 1 00000080:00200000:1.0:1775764786.008458:0:20498:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764786.008632:0:20498:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764786.008697:0:20498:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c82c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 20498 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764786.008708:0:20498:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c82c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 20498 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764786.008718:0:20498:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c82c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 20498 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764786.009152:0:20498:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:20498:1862022967024128:192.168.202.146@tcp:101:rm.0 00000100:00100000:1.0:1775764786.009310:0:20498:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f252840 going to sleep for 11 seconds 00000100:00100000:1.0:1775764786.012367:0:20498:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:20498:1862022967024128:192.168.202.146@tcp:101:rm.0 00010000:00010000:1.0:1775764786.012400:0:20498:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c82c lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb39b expref: -99 pid: 20498 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764786.012411:0:20498:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c82c lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb39b expref: -99 pid: 20498 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764786.012423:0:20498:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c82c lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb39b expref: -99 pid: 20498 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764786.012434:0:20498:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c82c lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb39b expref: -99 pid: 20498 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764786.012446:0:20498:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c82c lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb39b expref: -99 pid: 20498 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764786.012454:0:20498:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c82c lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb39b expref: -99 pid: 20498 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764786.012460:0:20498:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c82c lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb39b expref: -99 pid: 20498 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764786.012471:0:20498:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994f2e7b80 x1862022967024128/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764802 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764786.012662:0:20498:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c82c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb39b expref: -99 pid: 20498 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764786.012676:0:20498:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1775764786.012683:0:20498:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1775764786.012715:0:20498:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 8704 00000080:00010000:1.0:1775764786.012721:0:20498:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c82c 00000080:00010000:1.0:1775764786.012726:0:20498:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764786.012730:0:20498:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c82c from it ffffafbac176bb58 00010000:00010000:1.0:1775764786.012733:0:20498:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c82c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb39b expref: -99 pid: 20498 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764786.012793:0:20498:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c82c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb39b expref: -99 pid: 20498 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764786.012916:0:20498:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764786.012941:0:20498:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=fsa-oleg246-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8f9952d121c8), flags=8 00000080:00200000:1.0:1775764786.012946:0:20498:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=fsa-oleg246-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8f9952d121c8), intent=getattr 00000002:00010000:1.0:1775764786.013060:0:20498:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: fsa-oleg246-client.virtnet,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764786.013106:0:20498:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c833 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 20498 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764786.013114:0:20498:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c833 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 20498 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764786.013124:0:20498:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c833 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 20498 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764786.013145:0:20498:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:20498:1862022967024256:192.168.202.146@tcp:101:rm.0 00000100:00100000:1.0:1775764786.013237:0:20498:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f252840 going to sleep for 11 seconds 00000100:00100000:1.0:1775764786.025923:0:20498:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:20498:1862022967024256:192.168.202.146@tcp:101:rm.0 00010000:00010000:1.0:1775764786.025948:0:20498:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c833 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb3b7 expref: -99 pid: 20498 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764786.026786:0:20498:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c833 lrc: 4/1,0 mode: --/PR res: [0x200000401:0xeda:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb3b7 expref: -99 pid: 20498 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764786.026800:0:20498:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c833 lrc: 4/1,0 mode: --/PR res: [0x200000401:0xeda:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb3b7 expref: -99 pid: 20498 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764786.026883:0:20498:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c833 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xeda:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb3b7 expref: -99 pid: 20498 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764786.026900:0:20498:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c833 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xeda:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb3b7 expref: -99 pid: 20498 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764786.026917:0:20498:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c833 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xeda:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb3b7 expref: -99 pid: 20498 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764786.026926:0:20498:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c833 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xeda:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb3b7 expref: -99 pid: 20498 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764786.026932:0:20498:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c833 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xeda:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb3b7 expref: -99 pid: 20498 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764786.026945:0:20498:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994f2e7b80 x1862022967024256/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 600/584 e 0 to 0 dl 1775764802 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1775764786.026957:0:20498:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c833 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xeda:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb3b7 expref: -99 pid: 20498 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764786.028494:0:20498:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/3802 (0 0) 00000080:00200000:0.0:1775764786.028509:0:20498:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1775764786.028534:0:20498:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0xeda:0x0], updating i_size 0 00000080:00200000:0.0:1775764786.028537:0:20498:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0xeda:0x0](ffff8f9952eb5b88): rc = 0 00000080:00200000:0.0:1775764786.028588:0:20498:0:(file.c:6766:ll_layout_conf()) [0x200000401:0xeda:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:0.0:1775764786.028613:0:20498:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xeda:0x0](ffff8f9952eb5b88) for lock 0x36f9bdc9e421c833 00000080:00010000:0.0:1775764786.028626:0:20498:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xeda:0x0](ffff8f9952eb5b88) 00000080:00010000:0.0:1775764786.028629:0:20498:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c833 from it ffffafbac176bb20 00010000:00010000:0.0:1775764786.028634:0:20498:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c833 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xeda:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fb3b7 expref: -99 pid: 20498 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764786.028646:0:20498:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c833 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xeda:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fb3b7 expref: -99 pid: 20498 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:0.0:1775764786.028704:0:20498:0:(file.c:6226:ll_getattr_dentry()) START file fsa-oleg246-client.virtnet:[0x200000401:0xeda:0x0](ffff8f9952eb5b88), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1775764786.028710:0:20498:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xeda:0x0](ffff8f9952eb5b88),name=fsa-oleg246-client.virtnet 00000002:00010000:0.0:1775764786.028729:0:20498:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xeda:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764786.028768:0:20498:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c833 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xeda:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fb3b7 expref: -99 pid: 20498 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764786.028779:0:20498:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c833 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xeda:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fb3b7 expref: -99 pid: 20498 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764786.028787:0:20498:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764786.028791:0:20498:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xeda:0x0](ffff8f9952eb5b88) 00000080:00010000:0.0:1775764786.028794:0:20498:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xeda:0x0](ffff8f9952eb5b88) for lock 0x36f9bdc9e421c833 00000080:00010000:0.0:1775764786.028798:0:20498:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c833 from it ffffafbac176bcb8 00010000:00010000:0.0:1775764786.028801:0:20498:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c833 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xeda:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fb3b7 expref: -99 pid: 20498 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764786.028809:0:20498:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c833 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xeda:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fb3b7 expref: -99 pid: 20498 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764786.028871:0:20498:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:0.0:1775764786.028879:0:20498:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764786.028886:0:20498:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c82c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb39b expref: -99 pid: 20498 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764786.028894:0:20498:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c82c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb39b expref: -99 pid: 20498 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764786.028899:0:20498:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764786.028903:0:20498:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:0.0:1775764786.028905:0:20498:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c82c 00000080:00010000:0.0:1775764786.028909:0:20498:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c82c from it ffffafbac176bc10 00010000:00010000:0.0:1775764786.028911:0:20498:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c82c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb39b expref: -99 pid: 20498 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764786.028918:0:20498:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c82c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb39b expref: -99 pid: 20498 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764786.028925:0:20498:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:0.0:1775764786.028934:0:20498:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=fsa-oleg246-client.virtnet, flags=0 00000080:00200000:0.0:1775764786.028939:0:20498:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:0.0:1775764786.028945:0:20498:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764786.028950:0:20498:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c82c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb39b expref: -99 pid: 20498 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764786.028958:0:20498:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c82c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb39b expref: -99 pid: 20498 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764786.028975:0:20498:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764786.028979:0:20498:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:0.0:1775764786.028982:0:20498:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c82c 00000080:00010000:0.0:1775764786.028985:0:20498:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c82c from it ffffafbac176bd68 00010000:00010000:0.0:1775764786.028988:0:20498:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c82c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb39b expref: -99 pid: 20498 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764786.028994:0:20498:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c82c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb39b expref: -99 pid: 20498 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1775764786.029002:0:20498:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 3 00000080:00200000:0.0:1775764786.029008:0:20498:0:(namei.c:2395:ll_unlink()) VFS Op:name=fsa-oleg246-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8f9952d121c8) 00010000:00010000:0.0:1775764786.029045:0:20498:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c82c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fb39b expref: -99 pid: 20498 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764786.029070:0:20498:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c833 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xeda:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5fb3b7 expref: -99 pid: 20498 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764786.029088:0:20498:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0xeda:0x0]: losing layout lock 00010000:00010000:0.0:1775764786.029128:0:20498:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c833 lrc: 1/0,0 mode: --/PR res: [0x200000401:0xeda:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fb3b7 expref: -99 pid: 20498 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764786.029138:0:20498:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c82c lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fb39b expref: -99 pid: 20498 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764786.029143:0:20498:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1775764786.029145:0:20498:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c833 lrc: 0/0,0 mode: --/PR res: [0x200000401:0xeda:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fb3b7 expref: -99 pid: 20498 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764786.029161:0:20498:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c82c lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fb39b expref: -99 pid: 20498 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:0.0:1775764786.029178:0:20498:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1775764786.029190:0:20498:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970a77b80 x1862022967024512/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 512/4592 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764786.029213:0:20498:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77b80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:20498:1862022967024512:192.168.202.146@tcp:36:rm.0 00000100:00100000:0.0:1775764786.029308:0:20498:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9941254480 going to sleep for 11 seconds 00000100:00100000:3.0:1775764786.044571:0:20498:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77b80 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:20498:1862022967024512:192.168.202.146@tcp:36:rm.0 00000080:00200000:3.0:1775764786.044662:0:20498:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xeda:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764786.044672:0:20498:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xeda:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764786.044698:0:20498:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000401:0xeda:0x0](ffff8f9952eb5b88) 00010000:00010000:2.0:1775764786.694107:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764787.719114:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764788.742079:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:2.0:1775764789.443886:0:20528:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: local REPLAY BARRIER on lustre-MDT0000 00000080:00200000:3.0:1775764789.462289:0:20529:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:3.0:1775764789.462322:0:20529:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764789.462399:0:20529:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c83a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 20529 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.462410:0:20529:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c83a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 20529 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.462421:0:20529:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c83a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 20529 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764789.467583:0:20529:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job dd:3f83546c-01a9-4156-8a97-44c446245d3e:20529:1862022967024640:192.168.202.146@tcp:101:dd.0 00000100:00100000:3.0:1775764789.467843:0:20529:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9951fa9900 going to sleep for 11 seconds 00000100:00100000:3.0:1775764789.479943:0:20529:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job dd:3f83546c-01a9-4156-8a97-44c446245d3e:20529:1862022967024640:192.168.202.146@tcp:101:dd.0 00010000:00010000:3.0:1775764789.479970:0:20529:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c83a lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb3e1 expref: -99 pid: 20529 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.479982:0:20529:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c83a lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb3e1 expref: -99 pid: 20529 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.479993:0:20529:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c83a lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb3e1 expref: -99 pid: 20529 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.480004:0:20529:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c83a lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb3e1 expref: -99 pid: 20529 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.480017:0:20529:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c83a lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb3e1 expref: -99 pid: 20529 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.480032:0:20529:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c83a lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb3e1 expref: -99 pid: 20529 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.480040:0:20529:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c83a lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb3e1 expref: -99 pid: 20529 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764789.480053:0:20529:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970edf480 x1862022967024640/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764805 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1775764789.480069:0:20529:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c83a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb3e1 expref: -99 pid: 20529 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.480083:0:20529:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1775764789.480090:0:20529:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1775764789.480117:0:20529:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 8704 00000080:00010000:3.0:1775764789.480125:0:20529:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c83a 00000080:00010000:3.0:1775764789.480130:0:20529:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:3.0:1775764789.480132:0:20529:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c83a from it ffffafbac19bfbf8 00010000:00010000:3.0:1775764789.480135:0:20529:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c83a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb3e1 expref: -99 pid: 20529 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.480143:0:20529:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c83a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb3e1 expref: -99 pid: 20529 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764789.480179:0:20529:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:3.0:1775764789.480191:0:20529:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d9.sanity-pfl, flags=25 00000080:00200000:3.0:1775764789.480199:0:20529:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xed8:0x0](ffff8f9952eb4b08), inode mode 41ed mask 1 00000080:00200000:3.0:1775764789.480218:0:20529:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xed8:0x0](ffff8f9952eb4b08), inode mode 41ed mask 3 00000080:00200000:3.0:1775764789.480223:0:20529:0:(namei.c:1399:ll_atomic_open()) VFS Op:name=f9.sanity-pfl, dir=[0x200000401:0xed8:0x0](ffff8f9952eb4b08), file ffff8f996048c480, open_flags 8042, mode 81b6 opened 0 00000080:00200000:3.0:1775764789.480230:0:20529:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=f9.sanity-pfl, dir=[0x200000401:0xed8:0x0](ffff8f9952eb4b08), intent=open|creat 00000002:00010000:3.0:1775764789.480266:0:20529:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f9.sanity-pfl,[0x200000401:0xedb:0x0]) in obj [0x200000401:0xed8:0x0], intent: open|creat flags 0100103 00010000:00010000:3.0:1775764789.480320:0:20529:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c841 lrc: 3/0,1 mode: --/CW res: [0x200000401:0xed8:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 20529 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.480327:0:20529:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c841 lrc: 3/0,1 mode: --/CW res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 20529 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:3.0:1775764789.480343:0:20529:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1775764789.480349:0:20529:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970edf480 x1862022967024896/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4624/70288 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1775764789.480359:0:20529:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c841 lrc: 3/0,1 mode: --/CW res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 20529 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764789.480381:0:20529:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job dd:3f83546c-01a9-4156-8a97-44c446245d3e:20529:1862022967024896:192.168.202.146@tcp:101:dd.0 00000100:00100000:3.0:1775764789.480532:0:20529:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9951fa9900 going to sleep for 11 seconds 00000100:00100000:3.0:1775764789.494406:0:20529:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job dd:3f83546c-01a9-4156-8a97-44c446245d3e:20529:1862022967024896:192.168.202.146@tcp:101:dd.0 00010000:00010000:3.0:1775764789.494434:0:20529:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c841 lrc: 4/0,1 mode: --/CW res: [0x200000401:0xed8:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb3fd expref: -99 pid: 20529 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.494468:0:20529:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c841 lrc: 4/0,1 mode: --/PR res: [0x200000401:0xed9:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb3fd expref: -99 pid: 20529 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.494477:0:20529:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c841 lrc: 4/0,1 mode: --/PR res: [0x200000401:0xed9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb3fd expref: -99 pid: 20529 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.494494:0:20529:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c841 lrc: 4/0,1 mode: PR/PR res: [0x200000401:0xed9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb3fd expref: -99 pid: 20529 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.494505:0:20529:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c841 lrc: 4/0,1 mode: PR/PR res: [0x200000401:0xed9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb3fd expref: -99 pid: 20529 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.494519:0:20529:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c841 lrc: 5/1,1 mode: PR/PR res: [0x200000401:0xed9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb3fd expref: -99 pid: 20529 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.494527:0:20529:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c841 lrc: 5/1,1 mode: PR/PR res: [0x200000401:0xed9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb3fd expref: -99 pid: 20529 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.494534:0:20529:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c841 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xed9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb3fd expref: -99 pid: 20529 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764789.494545:0:20529:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=3 disposition=2b, status=0 req@ffff8f9970edf480 x1862022967024896/t4294997606(4294997606) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4624/848 e 0 to 0 dl 1775764805 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00100000:3.0:1775764789.494557:0:20529:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f9970edf480 x1862022967024896/t4294997606(4294997606) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4624/848 e 0 to 0 dl 1775764805 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1775764789.494570:0:20529:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 240 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c841 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xed9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb3fd expref: -99 pid: 20529 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:3.0:1775764789.494583:0:20529:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c841 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xed9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb3fd expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.494594:0:20529:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/3801 (0 0) 00000080:00200000:3.0:1775764789.494601:0:20529:0:(namei.c:943:ll_intent_lock()) intent lock 3 on i1 [0x200000401:0xed8:0x0] suppgids 0 -1: rc 0 00000080:00200000:3.0:1775764789.494689:0:20529:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0xed9:0x0](ffff8f9952ee9148): rc = 0 00000080:00200000:3.0:1775764789.494709:0:20529:0:(file.c:6766:ll_layout_conf()) [0x200000401:0xed9:0x0]: layout version change: 4294967294 -> 2 00000080:00010000:3.0:1775764789.494719:0:20529:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xed9:0x0](ffff8f9952ee9148) for lock 0x36f9bdc9e421c841 00000080:00010000:3.0:1775764789.494732:0:20529:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xed9:0x0](ffff8f9952ee9148) 00000080:00200008:3.0:1775764789.494794:0:20529:0:(file.c:1057:ll_file_open()) START file f9.sanity-pfl:[0x200000401:0xed9:0x0](ffff8f9952ee9148), flags 100102 00000002:00100000:3.0:1775764789.494816:0:20529:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f9970edf480 x1862022967024896/t4294997606(4294997606) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 768/848 e 0 to 0 dl 1775764805 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000080:00010000:3.0:1775764789.494861:0:20529:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c841 from it ffff8f99704b7900 00010000:00010000:3.0:1775764789.494865:0:20529:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c841 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xed9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fb3fd expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.494885:0:20529:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c841 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xed9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fb3fd expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764789.494907:0:20529:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xed9:0x0](ffff8f9952ee9148), inode mode 81a4 mask 46 00000080:00200008:3.0:1775764789.495206:0:20529:0:(llite_lib.c:2274:ll_setattr_raw()) START file f9.sanity-pfl:[0x200000401:0xed9:0x0](ffff8f9952ee9148) current size 0, valid attrs 2068, mode 0, uid 514, gid 0, new size 2097152, atime 1775764789.000000000, mtime 1775764789.000000000, ctime 1775764789.000000000 00010000:00010000:3.0:1775764789.495232:0:20529:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c841 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xed9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5fb3fd expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764789.495368:0:20529:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0xed9:0x0]: losing layout lock 00010000:00010000:3.0:1775764789.495424:0:20529:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c841 lrc: 1/0,0 mode: --/PR res: [0x200000401:0xed9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fb3fd expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764789.495431:0:20529:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764789.495434:0:20529:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c841 lrc: 0/0,0 mode: --/PR res: [0x200000401:0xed9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fb3fd expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000020:00100000:3.0:1775764789.495471:0:20529:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1775764789.495478:0:20529:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970ede680 x1862022967025152/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764789.495502:0:20529:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job dd:3f83546c-01a9-4156-8a97-44c446245d3e:20529:1862022967025152:192.168.202.146@tcp:36:dd.0 00000100:00100000:3.0:1775764789.495587:0:20529:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9951fa9900 going to sleep for 11 seconds 00000100:00100000:3.0:1775764789.501479:0:20529:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job dd:3f83546c-01a9-4156-8a97-44c446245d3e:20529:1862022967025152:192.168.202.146@tcp:36:dd.0 00000080:00200000:3.0:1775764789.501565:0:20529:0:(file.c:1815:ll_io_set_mirror()) f9.sanity-pfl: desiginated mirror: 0 00000080:00200000:3.0:1775764789.501572:0:20529:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xed9:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00010000:3.0:1775764789.501592:0:20529:0:(file.c:6988:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200000401:0xed9:0x0](ffff8f9952ee9148) 00000002:00010000:3.0:1775764789.501602:0:20529:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xed9:0x0]) in obj [0x200000401:0xed9:0x0], intent: layout flags 0 00010000:00010000:3.0:1775764789.501657:0:20529:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c848 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xed9:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 20529 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.501668:0:20529:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c848 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xed9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 20529 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.501680:0:20529:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c848 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xed9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 20529 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764789.501701:0:20529:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job dd:3f83546c-01a9-4156-8a97-44c446245d3e:20529:1862022967025280:192.168.202.146@tcp:101:dd.0 00000100:00100000:3.0:1775764789.501812:0:20529:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9951fa9900 going to sleep for 11 seconds 00000100:00100000:3.0:1775764789.504839:0:20529:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job dd:3f83546c-01a9-4156-8a97-44c446245d3e:20529:1862022967025280:192.168.202.146@tcp:101:dd.0 00010000:00010000:3.0:1775764789.504866:0:20529:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c848 lrc: 4/1,0 mode: CR/CR res: [0x200000401:0xed9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb412 expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.504887:0:20529:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c848 lrc: 4/1,0 mode: CR/CR res: [0x200000401:0xed9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb412 expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764789.504902:0:20529:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8f9970edce00 x1862022967025280/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 376/584 e 0 to 0 dl 1775764805 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1775764789.504913:0:20529:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 240 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c848 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xed9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb412 expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:3.0:1775764789.504924:0:20529:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c848 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xed9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb412 expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.505111:0:20529:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 16 res 8589935617/3801 (0 0) 00000080:00200000:3.0:1775764789.505117:0:20529:0:(namei.c:943:ll_intent_lock()) intent lock 1024 on i1 [0x200000401:0xed9:0x0] suppgids 0 0: rc 0 00000080:00010000:3.0:1775764789.505136:0:20529:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xed9:0x0](ffff8f9952ee9148) for lock 0x36f9bdc9e421c848 00000080:00010000:3.0:1775764789.505143:0:20529:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c848 from it ffffafbac19bfb08 00010000:00010000:3.0:1775764789.505148:0:20529:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c848 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xed9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb412 expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.505157:0:20529:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c848 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xed9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb412 expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764789.505179:0:20529:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c848 lrc: 2/1,0 mode: CR/CR res: [0x200000401:0xed9:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb412 expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764789.505191:0:20529:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c848 lrc: 2/1,0 mode: CR/CR res: [0x200000401:0xed9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb412 expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:3.0:1775764789.505204:0:20529:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200000401:0xed9:0x0](00000000b0b2eed9) being reconfigured ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c848 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xed9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb412 expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764789.505394:0:20529:0:(file.c:6766:ll_layout_conf()) [0x200000401:0xed9:0x0]: layout version change: 4294967294 -> 2 00010000:00010000:3.0:1775764789.505413:0:20529:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c848 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xed9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fb412 expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764789.505420:0:20529:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c848 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xed9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fb412 expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764789.505439:0:20529:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xed9:0x0] ignore/verify layout 0/0, layout version 2 need write layout 1, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764789.505443:0:20529:0:(vvp_io.c:355:vvp_io_fini()) [0x200000401:0xed9:0x0] write layout, type 3 [0x0, 0x200001) 00000080:00010000:3.0:1775764789.505482:0:20529:0:(file.c:6988:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200000401:0xed9:0x0](ffff8f9952ee9148) 00000002:00010000:3.0:1775764789.505491:0:20529:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xed9:0x0]) in obj [0x200000401:0xed9:0x0], intent: layout flags 02 00010000:00010000:3.0:1775764789.505505:0:20529:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c848 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xed9:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5fb412 expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764789.505527:0:20529:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0xed9:0x0]: losing layout lock 00010000:00010000:3.0:1775764789.505562:0:20529:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c848 lrc: 1/0,0 mode: --/CR res: [0x200000401:0xed9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fb412 expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764789.505569:0:20529:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764789.505572:0:20529:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4000/0x36f9bdc9e421c848 lrc: 0/0,0 mode: --/CR res: [0x200000401:0xed9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fb412 expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764789.505599:0:20529:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(EX) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c84f lrc: 3/0,1 mode: --/EX res: [0x200000401:0xed9:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 20529 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.505609:0:20529:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c84f lrc: 3/0,1 mode: --/EX res: [0x200000401:0xed9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 20529 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:3.0:1775764789.505619:0:20529:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1775764789.505625:0:20529:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970edce00 x1862022967025536/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 376/4440 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1775764789.505634:0:20529:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c84f lrc: 3/0,1 mode: --/EX res: [0x200000401:0xed9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 20529 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764789.505653:0:20529:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job dd:3f83546c-01a9-4156-8a97-44c446245d3e:20529:1862022967025536:192.168.202.146@tcp:101:dd.0 00000100:00100000:3.0:1775764789.505735:0:20529:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9951fa9900 going to sleep for 11 seconds 00000100:00100000:3.0:1775764789.515253:0:20529:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job dd:3f83546c-01a9-4156-8a97-44c446245d3e:20529:1862022967025536:192.168.202.146@tcp:101:dd.0 00010000:00010000:3.0:1775764789.515280:0:20529:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode CR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c84f lrc: 4/0,1 mode: --/EX res: [0x200000401:0xed9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb420 expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.515292:0:20529:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c84f lrc: 4/0,1 mode: --/CR res: [0x200000401:0xed9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb420 expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.515303:0:20529:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c84f lrc: 4/0,1 mode: CR/CR res: [0x200000401:0xed9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb420 expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.515313:0:20529:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c84f lrc: 4/0,1 mode: CR/CR res: [0x200000401:0xed9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb420 expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.515327:0:20529:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c84f lrc: 5/1,1 mode: CR/CR res: [0x200000401:0xed9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb420 expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.515338:0:20529:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(EX) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c84f lrc: 5/1,1 mode: CR/CR res: [0x200000401:0xed9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb420 expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.515344:0:20529:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c84f lrc: 4/1,0 mode: CR/CR res: [0x200000401:0xed9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb420 expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764789.515353:0:20529:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8f9970edce00 x1862022967025536/t4294997609(4294997609) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 376/584 e 0 to 0 dl 1775764805 ref 2 fl Complete:RQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1775764789.515371:0:20529:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 240 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c84f lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xed9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb420 expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:3.0:1775764789.515383:0:20529:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c84f lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xed9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb420 expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.515396:0:20529:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 16 res 8589935617/3801 (0 0) 00000080:00200000:3.0:1775764789.515401:0:20529:0:(namei.c:943:ll_intent_lock()) intent lock 1024 on i1 [0x200000401:0xed9:0x0] suppgids 0 0: rc 0 00000080:00010000:3.0:1775764789.515409:0:20529:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xed9:0x0](ffff8f9952ee9148) for lock 0x36f9bdc9e421c84f 00000080:00010000:3.0:1775764789.515417:0:20529:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c84f from it ffffafbac19bfb70 00010000:00010000:3.0:1775764789.515420:0:20529:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c84f lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xed9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb420 expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.515429:0:20529:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c84f lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xed9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb420 expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764789.515457:0:20529:0:(file.c:1815:ll_io_set_mirror()) f9.sanity-pfl: desiginated mirror: 0 00000080:00200000:3.0:1775764789.515465:0:20529:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xed9:0x0] ignore/verify layout 0/0, layout version 2 restore needed 0 00010000:00010000:3.0:1775764789.515476:0:20529:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c84f lrc: 2/1,0 mode: CR/CR res: [0x200000401:0xed9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb420 expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764789.515487:0:20529:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c84f lrc: 2/1,0 mode: CR/CR res: [0x200000401:0xed9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb420 expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:3.0:1775764789.515496:0:20529:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200000401:0xed9:0x0](00000000b0b2eed9) being reconfigured ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c84f lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xed9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb420 expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764789.515579:0:20529:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c84f lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xed9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb420 expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764789.515587:0:20529:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c84f lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xed9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb420 expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764789.515626:0:20529:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xed9:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1775764789.515639:0:20529:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764789.515661:0:20529:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:3.0:1775764789.515688:0:20529:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xed9:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1775764789.515711:0:20529:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c84f lrc: 2/1,0 mode: CR/CR res: [0x200000401:0xed9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb420 expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764789.515720:0:20529:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c84f lrc: 2/1,0 mode: CR/CR res: [0x200000401:0xed9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb420 expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:3.0:1775764789.515727:0:20529:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200000401:0xed9:0x0](00000000b0b2eed9) being reconfigured ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c84f lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xed9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb420 expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764789.515817:0:20529:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xed9:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1775764789.515822:0:20529:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764789.515835:0:20529:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:3.0:1775764789.515847:0:20529:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xed9:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764789.515907:0:20529:0:(file.c:6766:ll_layout_conf()) [0x200000401:0xed9:0x0]: layout version change: 4294967294 -> 3 00010000:00010000:3.0:1775764789.515917:0:20529:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c84f lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xed9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fb420 expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764789.515925:0:20529:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c84f lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xed9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fb420 expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00020000:00200000:3.0:1775764789.515940:0:20529:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x10 00000080:00200000:3.0:1775764789.515956:0:20529:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:3.0:1775764789.515974:0:20529:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764789.515978:0:20529:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f9951fa9930, osc lock ffff8f9966ace4c0, flags 0x0 00000008:00010000:3.0:1775764789.515982:0:20529:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764789.515984:0:20529:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f9951fa9970, osc lock ffff8f9966ace850, flags 0x0 00010000:00010000:3.0:1775764789.516024:0:20529:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000030032e1e/0x36f9bdc9e421c856 lrc: 3/0,1 mode: --/PW res: [0x280000403:0x660:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.516034:0:20529:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000030032e1e/0x36f9bdc9e421c856 lrc: 3/0,1 mode: --/PW res: [0x280000403:0x660:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.516043:0:20529:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000030032e1e/0x36f9bdc9e421c856 lrc: 3/0,1 mode: --/PW res: [0x280000403:0x660:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764789.516065:0:20529:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job dd:3f83546c-01a9-4156-8a97-44c446245d3e:20529:1862022967025664:192.168.202.146@tcp:101:dd.0 00000100:00100000:3.0:1775764789.516151:0:20529:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9951fa9300 going to sleep for 11 seconds 00000100:00100000:3.0:1775764789.523721:0:20529:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job dd:3f83546c-01a9-4156-8a97-44c446245d3e:20529:1862022967025664:192.168.202.146@tcp:101:dd.0 00010000:00010000:3.0:1775764789.523794:0:20529:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000030032e1e/0x36f9bdc9e421c856 lrc: 4/0,1 mode: --/PW res: [0x280000403:0x660:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb427 expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.523818:0:20529:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000030032e1e/0x36f9bdc9e421c856 lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x660:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb427 expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.523832:0:20529:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000030032e1e/0x36f9bdc9e421c856 lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x660:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb427 expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.523848:0:20529:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000030032e1e/0x36f9bdc9e421c856 lrc: 5/0,2 mode: PW/PW res: [0x280000403:0x660:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb427 expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:3.0:1775764789.523862:0:20529:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000030032e1e/0x36f9bdc9e421c856 lrc: 4/0,2 mode: PW/PW res: [0x280000403:0x660:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fb427 expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.523876:0:20529:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000030032e1e/0x36f9bdc9e421c856 lrc: 5/0,2 mode: PW/PW res: [0x280000403:0x660:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fb427 expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.523883:0:20529:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000030032e1e/0x36f9bdc9e421c856 lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x660:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fb427 expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.523961:0:20529:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000004a200161/0x36f9bdc9e421c85d lrc: 3/0,1 mode: --/PW res: [0x240000403:0x661:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.523970:0:20529:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000004a200161/0x36f9bdc9e421c85d lrc: 3/0,1 mode: --/PW res: [0x240000403:0x661:0x0].0x0 rrc: 2 type: EXT [1048576->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.523980:0:20529:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000004a200161/0x36f9bdc9e421c85d lrc: 3/0,1 mode: --/PW res: [0x240000403:0x661:0x0].0x0 rrc: 2 type: EXT [1048576->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764789.524002:0:20529:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job dd:3f83546c-01a9-4156-8a97-44c446245d3e:20529:1862022967025792:192.168.202.146@tcp:101:dd.0 00000100:00100000:3.0:1775764789.524074:0:20529:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9951fa9300 going to sleep for 11 seconds 00000100:00100000:3.0:1775764789.528787:0:20529:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job dd:3f83546c-01a9-4156-8a97-44c446245d3e:20529:1862022967025792:192.168.202.146@tcp:101:dd.0 00010000:00010000:3.0:1775764789.528813:0:20529:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000004a200161/0x36f9bdc9e421c85d lrc: 4/0,1 mode: --/PW res: [0x240000403:0x661:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb42e expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.528831:0:20529:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000004a200161/0x36f9bdc9e421c85d lrc: 4/0,1 mode: PW/PW res: [0x240000403:0x661:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb42e expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.528843:0:20529:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000004a200161/0x36f9bdc9e421c85d lrc: 4/0,1 mode: PW/PW res: [0x240000403:0x661:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb42e expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.528858:0:20529:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000004a200161/0x36f9bdc9e421c85d lrc: 5/0,2 mode: PW/PW res: [0x240000403:0x661:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb42e expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:3.0:1775764789.528872:0:20529:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000004a200161/0x36f9bdc9e421c85d lrc: 4/0,2 mode: PW/PW res: [0x240000403:0x661:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fb42e expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.528885:0:20529:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000004a200161/0x36f9bdc9e421c85d lrc: 5/0,2 mode: PW/PW res: [0x240000403:0x661:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fb42e expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764789.528893:0:20529:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000004a200161/0x36f9bdc9e421c85d lrc: 4/0,1 mode: PW/PW res: [0x240000403:0x661:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fb42e expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764789.529049:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967025920:192.168.202.146@tcp:10:dd.0 00000100:00100000:2.0:1775764789.555613:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967025920:192.168.202.146@tcp:10:dd.0 00000080:00200000:1.0:1775764789.556321:0:20529:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0xed9:0x0] updating i_size 2097152 i_blocks 2 00000020:00010000:1.0:1775764789.556330:0:20529:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99587d5d58 (ffff8f9970314518/0) at cl_lock_release():245 00000020:00010000:1.0:1775764789.556333:0:20529:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99587d5d58 (ffff8f9970314518/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764789.556338:0:20529:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9951fa9930 (ffff8f9970314e18/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764789.556350:0:20529:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000030032e1e/0x36f9bdc9e421c856 lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x660:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fb427 expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764789.556371:0:20529:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000030032e1e/0x36f9bdc9e421c856 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x660:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fb427 expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:1.0:1775764789.556391:0:20529:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9951fa9970 (ffff8f9970314f38/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764789.556397:0:20529:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000004a200161/0x36f9bdc9e421c85d lrc: 4/0,1 mode: PW/PW res: [0x240000403:0x661:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fb42e expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764789.556405:0:20529:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000004a200161/0x36f9bdc9e421c85d lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x661:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fb42e expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:1.0:1775764789.556415:0:20529:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99587d5d58 (ffff8f9970314518/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764789.556419:0:20529:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9951fa9930 (ffff8f9970314518/1) at cl_lock_fini():72 00000020:00010000:1.0:1775764789.556424:0:20529:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9951fa9970 (ffff8f9970314518/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764789.556486:0:20529:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xed9:0x0] ignore/verify layout 0/0, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1775764789.556506:0:20529:0:(llite_lib.c:2524:ll_setattr_raw()) COMPLETED file f9.sanity-pfl:[0x200000401:0xed9:0x0](ffff8f9952ee9148) current size 2097152, valid attrs 2168, mode 0, uid 514, gid 0, new size 2097152, atime 1775764789.000000000, mtime 1775764789.000000000, ctime 1775764789.000000000, rc 0 00000080:00200080:1.0:1775764789.556565:0:20529:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200000401:0xed9:0x0](ffff8f9952ee9148) cmd=80306d02 arg=7ffd3db1c960 00020000:00000080:1.0:1775764789.556576:0:20529:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=80306d02 len=0 karg=0000000000000000 uarg=00007ffd3db1c960 00000020:00000080:1.0:1775764789.556581:0:20529:0:(lov_obd.c:966:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: iocontrol from 'dd' cmd=80306d02 _IOR('m', 2, 48) unknown: rc = -25 00000080:00200008:1.0:1775764789.556589:0:20529:0:(file.c:5170:ll_file_seek()) START file f9.sanity-pfl:[0x200000401:0xed9:0x0], offset: 2097152, type: SEEK_CUR 00000080:00200008:1.0:1775764789.556597:0:20529:0:(file.c:5201:ll_file_seek()) COMPLETED file f9.sanity-pfl:[0x200000401:0xed9:0x0], offset: 2097152, type: SEEK_CUR, rc = 2097152 00000080:00200008:1.0:1775764789.556657:0:20529:0:(file.c:2612:do_file_write_iter()) START file f9.sanity-pfl:[0x200000401:0xed9:0x0], ppos: 2097152, count: 1024 00000080:00200000:1.0:1775764789.556672:0:20529:0:(rw26.c:663:ll_write_begin()) Writing 512 of 0 to 1024 bytes 00000080:00200000:1.0:1775764789.556679:0:20529:0:(file.c:2588:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:1.0:1775764789.556710:0:20529:0:(file.c:2058:ll_file_io_generic()) f9.sanity-pfl: write ppos: 2097152, bytes: 1024 00000080:00200000:1.0:1775764789.556715:0:20529:0:(file.c:1815:ll_io_set_mirror()) f9.sanity-pfl: desiginated mirror: 0 00000020:00200000:1.0:1775764789.556721:0:20529:0:(cl_io.c:214:cl_io_rw_init()) header@0000000004134823[0x110, 1, [0x200000401:0xed9:0x0] hash] 00000020:00200000:1.0:1775764789.556723:0:20529:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [2097152, 2098176) 0 0 00000080:00200000:1.0:1775764789.556726:0:20529:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xed9:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764789.558170:0:20529:0:(file.c:2136:ll_file_io_generic()) Range lock [512, 512] 00020000:00200000:1.0:1775764789.558184:0:20529:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 2097152 chunk: [2097152, 2098176) 2098176, 1024 00020000:00200000:1.0:1775764789.558188:0:20529:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x10 00000080:00200000:1.0:1775764789.558321:0:20529:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [512, 512] 00000008:00010000:1.0:1775764789.558337:0:20529:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764789.558342:0:20529:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f1c7130, osc lock ffff8f995875d690, flags 0x0 00010000:00010000:1.0:1775764789.558365:0:20529:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000004a200161/0x36f9bdc9e421c85d lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x661:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fb42e expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764789.558382:0:20529:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (2097152 2101247) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000004a200161/0x36f9bdc9e421c85d lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x661:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fb42e expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764789.558395:0:20529:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000004a200161/0x36f9bdc9e421c85d lrc: 6/0,2 mode: PW/PW res: [0x240000403:0x661:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fb42e expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764789.558403:0:20529:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000004a200161/0x36f9bdc9e421c85d lrc: 6/0,2 mode: PW/PW res: [0x240000403:0x661:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fb42e expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764789.558411:0:20529:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000004a200161/0x36f9bdc9e421c85d lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x661:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fb42e expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764789.558422:0:20529:0:(vvp_io.c:1286:vvp_io_write_start()) f9.sanity-pfl: write [2097152, 2098176) append 0 append lockpos 0 00000080:00200000:1.0:1775764789.558435:0:20529:0:(rw26.c:663:ll_write_begin()) Writing 512 of 0 to 1024 bytes 00000080:00200000:1.0:1775764789.558503:0:20529:0:(rw26.c:858:ll_write_end()) pos 2097152, len 1024, copied 1024 00000080:00200000:1.0:1775764789.558512:0:20529:0:(rw26.c:916:ll_write_end()) page@000000006e06c3d9[3 0000000044d00fc7 2 1 000000008ee81b18] 00000080:00200000:1.0:1775764789.558516:0:20529:0:(rw26.c:916:ll_write_end()) vmpage @000000008fdcb911 17ffffc0001001 3:0 ffff8f994ee73700 512 lru 00000080:00200000:1.0:1775764789.558523:0:20529:0:(rw26.c:916:ll_write_end()) osc-page@000000000fe34b36 512: 1< 0 - - > 2< 2097152 0 0 0x0 0x0 | 00000000f240ca6f 00000000cb42a053 > 3< 0 0 > 4< 0 0 8 410648576 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764789.558526:0:20529:0:(rw26.c:916:ll_write_end()) end page@000000006e06c3d9 00000080:00200000:1.0:1775764789.558528:0:20529:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764789.558532:0:20529:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:1.0:1775764789.558552:0:20529:0:(osc_cache.c:848:osc_extent_find()) ### extent: 00000000c658c123 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000004a200161/0x36f9bdc9e421c85d lrc: 4/0,1 mode: PW/PW res: [0x240000403:0x661:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fb42e expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764789.558572:0:20529:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8f9952ee92f8, count 1, dirtied 1 00000080:00200000:1.0:1775764789.558576:0:20529:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:1.0:1775764789.558579:0:20529:0:(vvp_io.c:1353:vvp_io_write_start()) f9.sanity-pfl: write bytes 0, result: 1024 00000020:00010000:1.0:1775764789.558594:0:20529:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705ba998 (ffff8f99705b8878/0) at cl_lock_release():245 00000020:00010000:1.0:1775764789.558597:0:20529:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705ba998 (ffff8f99705b8878/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764789.558600:0:20529:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f1c7130 (ffff8f99705b83f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764789.558604:0:20529:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000004a200161/0x36f9bdc9e421c85d lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x661:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fb42e expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764789.559199:0:20529:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000004a200161/0x36f9bdc9e421c85d lrc: 4/0,0 mode: PW/PW res: [0x240000403:0x661:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fb42e expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00010000:1.0:1775764789.559389:0:20529:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705ba998 (ffff8f99705b8878/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764789.559393:0:20529:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f1c7130 (ffff8f99705b8878/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764789.559405:0:20529:0:(vvp_io.c:517:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:1.0:1775764789.559412:0:20529:0:(file.c:2173:ll_file_io_generic()) Range unlock [512, 512] 00000080:00200000:1.0:1775764789.559444:0:20529:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xed9:0x0] ignore/verify layout 0/0, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764789.559456:0:20529:0:(file.c:2206:ll_file_io_generic()) f9.sanity-pfl: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:1.0:1775764789.559466:0:20529:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0xed9:0x0] updating i_size 2098176 i_blocks 2 00000080:00200000:1.0:1775764789.559473:0:20529:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:1.0:1775764789.559527:0:20529:0:(file.c:447:ll_file_release()) START file f9.sanity-pfl:[0x200000401:0xed9:0x0](ffff8f9952ee9148), flags 100002 00000002:00100000:1.0:1775764789.559604:0:20529:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f9970edf480 x1862022967024896/t4294997606(4294997606) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 768/848 e 0 to 0 dl 1775764805 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dd.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1775764789.559628:0:20529:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764789.559633:0:20529:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e5500 x1862022967026176/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764789.559656:0:20529:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5500 pname:cluuid:pid:xid:nid:opc:job dd:3f83546c-01a9-4156-8a97-44c446245d3e:20529:1862022967026176:192.168.202.146@tcp:35:dd.0 00000100:00100000:1.0:1775764789.559735:0:20529:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f2529c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764789.565470:0:20529:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5500 pname:cluuid:pid:xid:nid:opc:job dd:3f83546c-01a9-4156-8a97-44c446245d3e:20529:1862022967026176:192.168.202.146@tcp:35:dd.0 00000002:00100000:2.0:1775764789.565509:0:20529:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f9970edf480 x1862022967024896/t4294997606(4294997606) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 768/848 e 0 to 0 dl 1775764805 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dd.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764789.585236:0:20531:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764789.585267:0:20531:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764789.585289:0:20531:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c83a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb3e1 expref: -99 pid: 20529 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764789.585304:0:20531:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c83a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb3e1 expref: -99 pid: 20529 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764789.585312:0:20531:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764789.585321:0:20531:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764789.585325:0:20531:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c83a 00000080:00010000:1.0:1775764789.585332:0:20531:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c83a from it ffffafbac19d7bf8 00010000:00010000:1.0:1775764789.585335:0:20531:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c83a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb3e1 expref: -99 pid: 20529 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764789.585344:0:20531:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c83a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb3e1 expref: -99 pid: 20529 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764789.585360:0:20531:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764789.585369:0:20531:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d9.sanity-pfl, flags=27 00000080:00200000:1.0:1775764789.585374:0:20531:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xed8:0x0](ffff8f9952eb4b08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764789.585385:0:20531:0:(namei.c:1399:ll_atomic_open()) VFS Op:name=f9.sanity-pfl, dir=[0x200000401:0xed8:0x0](ffff8f9952eb4b08), file ffff8f99427e3500, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1775764789.585392:0:20531:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=f9.sanity-pfl, dir=[0x200000401:0xed8:0x0](ffff8f9952eb4b08), intent=open 00000002:00010000:1.0:1775764789.585406:0:20531:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f9.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000401:0xed8:0x0], intent: open flags 0304001 00010000:00010000:1.0:1775764789.591570:0:20531:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c864 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 20531 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764789.591590:0:20531:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c864 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 20531 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764789.591605:0:20531:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764789.591628:0:20531:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e6300 x1862022967026432/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4624/135824 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764789.591638:0:20531:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c864 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 20531 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764789.591661:0:20531:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:20531:1862022967026432:192.168.202.146@tcp:101:lfs.0 00000100:00100000:1.0:1775764789.592022:0:20531:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994f252600 going to sleep for 11 seconds 00000100:00100000:2.0:1775764789.606051:0:20531:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:20531:1862022967026432:192.168.202.146@tcp:101:lfs.0 00010000:00010000:2.0:1775764789.606072:0:20531:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c864 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 20531 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764789.606085:0:20531:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c864 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 20531 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764789.606104:0:20531:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c864 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 20531 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764789.606111:0:20531:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c864 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 20531 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764789.606119:0:20531:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c864 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 20531 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764789.606125:0:20531:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970640200 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:2.0:1775764789.606130:0:20531:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c864 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 20531 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764789.606137:0:20531:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c864 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 20531 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764789.606143:0:20531:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c864 lrc: 5/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 20531 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764789.606150:0:20531:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1775764789.606157:0:20531:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c864 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 20531 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764789.606163:0:20531:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c864 lrc: 0/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 20531 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764789.606180:0:20531:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff8f994f2e6300 x1862022967026432/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4624/608 e 0 to 0 dl 1775764805 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764789.606193:0:20531:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0xed8:0x0] suppgids 0 -1: rc -20 00000080:00200000:2.0:1775764789.606303:0:20531:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:2.0:1775764789.606318:0:20531:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764789.606330:0:20531:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c83a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb3e1 expref: -99 pid: 20529 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764789.606338:0:20531:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c83a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb3e1 expref: -99 pid: 20529 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764789.606344:0:20531:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764789.606348:0:20531:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:2.0:1775764789.606350:0:20531:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c83a 00000080:00010000:2.0:1775764789.606354:0:20531:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c83a from it ffffafbac19d7bf8 00010000:00010000:2.0:1775764789.606356:0:20531:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c83a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fb3e1 expref: -99 pid: 20529 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764789.606361:0:20531:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c83a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fb3e1 expref: -99 pid: 20529 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764789.606370:0:20531:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:2.0:1775764789.606374:0:20531:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d9.sanity-pfl, flags=267 00000080:00200000:2.0:1775764789.606379:0:20531:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xed8:0x0](ffff8f9952eb4b08), inode mode 41ed mask 44 00000080:00200008:2.0:1775764789.606384:0:20531:0:(file.c:1057:ll_file_open()) START file d9.sanity-pfl:[0x200000401:0xed8:0x0](ffff8f9952eb4b08), flags 304000 00000002:00010000:2.0:1775764789.606394:0:20531:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xed8:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100240304001 00010000:00010000:2.0:1775764789.606428:0:20531:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c86b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 20531 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764789.606433:0:20531:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c86b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 20531 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764789.606440:0:20531:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764789.606455:0:20531:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f99703d1500 x1862022967026688/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/135824 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764789.606466:0:20531:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c86b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 20531 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764789.606493:0:20531:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:20531:1862022967026688:192.168.202.146@tcp:101:lfs.0 00000100:00100000:2.0:1775764789.606664:0:20531:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560a80 going to sleep for 11 seconds 00000100:00100000:2.0:1775764789.621933:0:20531:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:20531:1862022967026688:192.168.202.146@tcp:101:lfs.0 00010000:00010000:2.0:1775764789.621959:0:20531:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c86b lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 20531 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764789.621972:0:20531:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c86b lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 20531 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764789.621980:0:20531:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c86b lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 20531 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764789.621987:0:20531:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c86b lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 20531 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764789.621997:0:20531:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c86b lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 20531 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764789.622015:0:20531:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99706ea400 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:2.0:1775764789.622021:0:20531:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c86b lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 20531 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764789.622028:0:20531:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c86b lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 20531 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764789.622035:0:20531:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c86b lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 20531 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764789.622042:0:20531:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1775764789.622048:0:20531:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c86b lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 20531 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764789.622054:0:20531:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421c86b lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 20531 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764789.622073:0:20531:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8f99703d1500 x1862022967026688/t4294997611(4294997611) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764805 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764789.622097:0:20531:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f99703d1500 x1862022967026688/t4294997611(4294997611) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764805 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764789.622113:0:20531:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764789.622135:0:20531:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0xed8:0x0], updating i_size 7680 00000002:00100000:2.0:1775764789.622153:0:20531:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f99703d1500 x1862022967026688/t4294997611(4294997611) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764805 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:2.0:1775764789.622260:0:20531:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000401:0xed8:0x0](ffff8f9952eb4b08) cmd=c0086915 arg=169c3d0 00000100:00100000:2.0:1775764789.622328:0:20531:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:20531:1862022967026816:192.168.202.146@tcp:34:lfs.0 00000100:00100000:2.0:1775764789.622417:0:20531:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560a80 going to sleep for 11 seconds 00000100:00100000:2.0:1775764789.632858:0:20531:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:20531:1862022967026816:192.168.202.146@tcp:34:lfs.0 00000080:00200008:2.0:1775764789.633140:0:20531:0:(file.c:447:ll_file_release()) START file d9.sanity-pfl:[0x200000401:0xed8:0x0](ffff8f9952eb4b08), flags 304000 00000002:00100000:2.0:1775764789.633215:0:20531:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f99703d1500 x1862022967026688/t4294997611(4294997611) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764805 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764789.633226:0:20531:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764789.633424:0:20531:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f99703d2680 x1862022967027072/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/4592 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764789.633458:0:20531:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:20531:1862022967027072:192.168.202.146@tcp:35:lfs.0 00000100:00100000:2.0:1775764789.633608:0:20531:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560a80 going to sleep for 11 seconds 00000100:00100000:2.0:1775764789.645395:0:20531:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:20531:1862022967027072:192.168.202.146@tcp:35:lfs.0 00000002:00100000:2.0:1775764789.645435:0:20531:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f99703d1500 x1862022967026688/t4294997611(4294997611) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 4616/608 e 0 to 0 dl 1775764805 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00010000:00010000:1.0:1775764789.781509:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764789.958315:0:2382:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1775764789.958338:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2382:1862022967027200:192.168.202.146@tcp:400:kworker.0 00000100:00100000:2.0:1775764789.972830:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2382:1862022967027200:192.168.202.146@tcp:400:kworker.0 00010000:00010000:0.0:1775764790.805647:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764791.814145:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764792.838167:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764793.862091:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1775764794.570122:0:38:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xed9:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764794.570138:0:38:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1775764794.570159:0:38:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x10 00000008:00010000:0.0:1775764794.570217:0:38:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### extent: 00000000c658c123 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000004a200161/0x36f9bdc9e421c85d lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x661:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fb42e expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:0.0:1775764794.570241:0:38:0:(osc_cache.c:1167:osc_extent_make_ready()) ### extent: 00000000c658c123 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000004a200161/0x36f9bdc9e421c85d lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x661:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fb42e expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764794.570964:0:38:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (2097152 2101247) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000004a200161/0x36f9bdc9e421c85d lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x661:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fb42e expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1775764794.571143:0:38:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8f9970a77800 - object 0x240000403:1633 offset 2097152<>2098176 00000100:00100000:1.0:1775764794.571257:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967027328:192.168.202.146@tcp:4:dd.0 00000100:00100000:1.0:1775764794.585527:0:2380:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1775764794.585555:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967027456:192.168.202.146@tcp:16:kworker.0 00000100:00100000:0.0:1775764794.590644:0:2380:0:(client.c:3082:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8f99523ee800: committing for last_committed 4294971562 gen 5 00000100:00100000:0.0:1775764794.590676:0:2380:0:(client.c:3126:ptlrpc_free_committed()) @@@ stopping search req@ffff8f9970a77800 x1862022967027328/t4294971563(4294971563) o4->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:6/4 lens 1512/448 e 0 to 0 dl 1775764810 ref 3 fl Rpc:RQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000008:00010000:0.0:1775764794.590722:0:2380:0:(osc_cache.c:901:osc_extent_finish()) ### extent: 00000000c658c123 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000004a200161/0x36f9bdc9e421c85d lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x661:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fb42e expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1775764794.590788:0:2380:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000c658c123 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000004a200161/0x36f9bdc9e421c85d lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x661:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fb42e expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1775764794.590816:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967027328:192.168.202.146@tcp:4:dd.0 00000100:00100000:0.0:1775764794.627446:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967027456:192.168.202.146@tcp:16:kworker.0 00000080:00200000:1.0:1775764794.627585:0:38:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xed9:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:1.0:1775764794.886310:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764795.080541:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2380:1862022967027584:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764795.080641:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967027840:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764795.080685:0:2380:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1775764795.080691:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967027712:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764795.087110:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967027840:192.168.202.146@tcp:400:kworker.0 00010000:00010000:1.0:1775764795.910202:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764796.935369:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764797.959520:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764798.984016:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764800.006216:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764800.198431:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967028096:192.168.202.146@tcp:400:kworker.0 00000100:00100000:2.0:1775764800.198650:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2382:1862022967027968:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764800.198708:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967028352:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764800.213817:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967028352:192.168.202.146@tcp:400:kworker.0 00000100:00100000:2.0:1775764800.220833:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967028224:192.168.202.146@tcp:400:kworker.0 00000100:00100000:2.0:1775764800.224043:0:2382:0:(client.c:3082:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8f99523ee800: committing for last_committed 4294971563 gen 5 00000100:00100000:2.0:1775764800.224080:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967028224:192.168.202.146@tcp:400:kworker.0 00010000:00010000:1.0:1775764801.032218:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764802.059655:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764803.078828:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764804.109903:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764805.155005:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764805.325380:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2382:1862022967028480:192.168.202.146@tcp:400:kworker.0 00000100:00100000:2.0:1775764805.325501:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967028736:192.168.202.146@tcp:400:kworker.0 00000100:00100000:2.0:1775764805.325549:0:2382:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:2.0:1775764805.325556:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967028608:192.168.202.146@tcp:400:kworker.0 00000100:00100000:2.0:1775764805.325580:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967028864:192.168.202.146@tcp:400:kworker.0 00000100:00100000:2.0:1775764805.342978:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967028864:192.168.202.146@tcp:400:kworker.0 00000100:00100000:2.0:1775764805.342996:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967028736:192.168.202.146@tcp:400:kworker.0 00010000:00010000:1.0:1775764806.214150:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764807.238067:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764808.262039:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764809.288369:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764810.313535:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764810.438120:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967029120:192.168.202.146@tcp:400:kworker.0 00000100:00100000:2.0:1775764810.438229:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967029376:192.168.202.146@tcp:400:kworker.0 00000100:00100000:2.0:1775764810.438281:0:2381:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1775764810.438288:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:884ae1b0-72f8-4a88-a129-db99cded7efc:2381:1862022967028992:192.168.202.146@tcp:400:kworker.0 00000100:00100000:2.0:1775764810.438316:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967029248:192.168.202.146@tcp:400:kworker.0 00000100:00100000:2.0:1775764810.440951:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967029248:192.168.202.146@tcp:400:kworker.0 00000100:00100000:2.0:1775764810.440972:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967029376:192.168.202.146@tcp:400:kworker.0 00010000:00010000:0.0:1775764811.334727:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:1.0:1775764811.461885:0:2380:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764795/real 1775764795] req@ffff8f9970a77480 x1862022967027712/t0(0) o400->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764811 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764811.500929:0:2380:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8f9970a77480 x1862022967027712/t0(0) o400->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764811 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764811.500944:0:2380:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:1.0:1775764811.500949:0:2380:0:(import.c:181:ptlrpc_set_import_discon()) lustre-MDT0000-mdc-ffff8f99523ee800: Connection to lustre-MDT0000 (at 192.168.202.146@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:1.0:1775764811.535787:0:2380:0:(import.c:416:ptlrpc_pinger_force()) lustre-MDT0000_UUID: waking up pinger s:DISCONN 00000100:00000400:1.0:1775764811.535834:0:2380:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764795/real 1775764795] req@ffff8f9970a75c00 x1862022967027584/t0(0) o400->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 224/224 e 0 to 1 dl 1775764811 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764811.535869:0:2380:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8f9970a75c00 x1862022967027584/t0(0) o400->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 224/224 e 0 to 1 dl 1775764811 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764811.535877:0:2380:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1775764811.535881:0:2380:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.202.146@tcp: Connection to MGS (at 192.168.202.146@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1775764811.560511:0:2380:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.202.146@tcp for MGC192.168.202.146@tcp not replayable, auto-deactivating 00000100:00080000:1.0:1775764811.560517:0:2380:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00100000:1.0:1775764811.560527:0:2380:0:(client.c:3560:ptlrpc_abort_inflight()) @@@ inflight req@ffff8f9970a75c00 x1862022967027584/t0(0) o400->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 224/224 e 0 to 1 dl 1775764811 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764811.560544:0:2380:0:(client.c:3560:ptlrpc_abort_inflight()) @@@ inflight req@ffff8f9970ede300 x1862022967027968/t0(0) o400->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 224/224 e 0 to 0 dl 1775764816 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764811.561066:0:2380:0:(client.c:3560:ptlrpc_abort_inflight()) @@@ inflight req@ffff8f99703d2680 x1862022967028480/t0(0) o400->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 224/224 e 0 to 0 dl 1775764821 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764811.561077:0:2380:0:(client.c:3560:ptlrpc_abort_inflight()) @@@ inflight req@ffff8f99703d3b80 x1862022967028992/t0(0) o400->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 224/224 e 0 to 0 dl 1775764826 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764811.561141:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764811.561146:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775764811.561150:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775764811.561156:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 253 00000100:00080000:2.0:1775764811.561196:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775764811.561234:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:884ae1b0-72f8-4a88-a129-db99cded7efc:2381:1862022967028992:192.168.202.146@tcp:400:kworker.0 00000100:00100000:2.0:1775764811.561388:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3b80 x1862022967029504/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775764811.561408:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764811.561412:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775764811.561414:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775764811.561436:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 253 00000100:00080000:2.0:1775764811.561443:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775764811.561455:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775764811.561468:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2d80 x1862022967029632/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764811.561499:0:2381:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1775764811.561516:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967029760:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764811.561865:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967029504:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775764811.561932:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967029632:192.168.202.146@tcp:38:kworker.0 10000000:01000000:1.0:1775764811.562777:0:2380:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1775764811.562782:0:2380:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:01000000:1.0:1775764811.562783:0:2380:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 00000100:00080000:1.0:1775764811.562786:0:2380:0:(import.c:416:ptlrpc_pinger_force()) MGS: waking up pinger s:CONNECTING 00000100:00100000:2.0:1775764811.564699:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967029888:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764811.566368:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967027712:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764811.566379:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2380:1862022967027584:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764811.568024:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2382:1862022967027968:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764811.568054:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2382:1862022967028480:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764811.571476:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967029888:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764811.571492:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967029760:192.168.202.146@tcp:400:kworker.0 00000100:00080000:2.0:1775764811.571573:0:2378:0:(import.c:1130:ptlrpc_connect_interpret()) MGC192.168.202.146@tcp: connect to target with instance 0 10000000:01000000:2.0:1775764811.571587:0:2378:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 00000100:02000400:2.0:1775764811.571592:0:2378:0:(import.c:1282:ptlrpc_connect_interpret()) Evicted from MGS (at 192.168.202.146@tcp) after server handle changed from 0x2aac6a50e4d8202 to 0x2aac6a50e5fb489 00000100:00080000:2.0:1775764811.580140:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to EVICTED 00000100:00080000:2.0:1775764811.580157:0:2378:0:(import.c:1667:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.202.146@tcp; invalidating 00000100:00100000:2.0:1775764811.581783:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967029504:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775764811.581848:0:20719:0:(import.c:1591:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.202.146@tcp to MGS@192.168.202.146@tcp 00000100:00100000:1.0:1775764811.581854:0:20719:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 10000000:01000000:1.0:1775764811.581858:0:20719:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00010000:00010000:1.0:1775764811.581875:0:20719:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.146@tcp lock: ffff8f9940030e00/0x36f9bdc9e41c4478 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x110e400000000 nid: local remote: 0x2aac6a50e4d82b1 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:1.0:1775764811.581885:0:20719:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.146@tcp lock: ffff8f9940030e00/0x36f9bdc9e41c4478 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x110f400000000 nid: local remote: 0x2aac6a50e4d82b1 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1775764811.581891:0:20719:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:1.0:1775764811.581985:0:20719:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1775764811.581995:0:20719:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.146@tcp lock: ffff8f9940030e00/0x36f9bdc9e41c4478 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x590f400000000 nid: local remote: 0x2aac6a50e4d82b1 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764811.582024:0:20719:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.146@tcp lock: ffff8f99820e3800/0x36f9bdc9e41c447f lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0x2aac6a50e4d82b8 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:1.0:1775764811.582031:0:20719:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.146@tcp lock: ffff8f99820e3800/0x36f9bdc9e41c447f lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0x2aac6a50e4d82b8 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1775764811.582037:0:20719:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:1.0:1775764811.582338:0:20719:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1775764811.582344:0:20719:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.146@tcp lock: ffff8f99820e3800/0x36f9bdc9e41c447f lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0x2aac6a50e4d82b8 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764811.582360:0:20719:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.146@tcp lock: ffff8f9940031c00/0x36f9bdc9e41c4440 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0x2aac6a50e4d8217 expref: -99 pid: 5560 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:1.0:1775764811.582366:0:20719:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.146@tcp lock: ffff8f9940031c00/0x36f9bdc9e41c4440 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0x2aac6a50e4d8217 expref: -99 pid: 5560 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1775764811.582371:0:20719:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 00010000:00010000:1.0:1775764811.582448:0:20719:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1775764811.582454:0:20719:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.146@tcp lock: ffff8f9940031c00/0x36f9bdc9e41c4440 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0x2aac6a50e4d8217 expref: -99 pid: 5560 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764811.582468:0:20719:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.146@tcp lock: ffff8f9940030a00/0x36f9bdc9e41c448d lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0x2aac6a50e4d8328 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:1.0:1775764811.582474:0:20719:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.146@tcp lock: ffff8f9940030a00/0x36f9bdc9e41c448d lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0x2aac6a50e4d8328 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1775764811.582479:0:20719:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x736d61726170:0x3:0x0].0x0 (params) 00010000:00010000:1.0:1775764811.582561:0:20719:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1775764811.582672:0:20719:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.146@tcp lock: ffff8f9940030a00/0x36f9bdc9e41c448d lrc: 0/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0x2aac6a50e4d8328 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00080000:1.0:1775764811.582690:0:20719:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from EVICTED to RECOVER 00000100:00080000:1.0:1775764811.582696:0:20719:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from RECOVER to FULL 10000000:01000000:1.0:1775764811.582699:0:20719:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 00000100:00080000:1.0:1775764811.582799:0:20719:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 00000100:02000000:1.0:1775764811.582805:0:20719:0:(import.c:1737:ptlrpc_import_recovery_state_machine()) MGC192.168.202.146@tcp: Connection restored to 192.168.202.146@tcp (at 192.168.202.146@tcp) 00010000:00010000:1.0:1775764812.358090:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:01000000:0.0:1775764812.805910:0:5575:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:01000000:0.0:1775764812.805918:0:5575:0:(mgc_request.c:1819:mgc_process_log()) Process log lustre-cliir-ffff8f99523ee800 from 6 10000000:01000000:0.0:1775764812.805923:0:5575:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00010000:00010000:0.0:1775764812.805979:0:5575:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.146@tcp lock: ffff8f99821cc600/0x36f9bdc9e421c872 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764812.805989:0:5575:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.146@tcp lock: ffff8f99821cc600/0x36f9bdc9e421c872 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764812.805999:0:5575:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.146@tcp lock: ffff8f99821cc600/0x36f9bdc9e421c872 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764812.806026:0:5575:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967030016:192.168.202.146@tcp:101:ll_cfg_requeue.0 00000100:00100000:0.0:1775764812.806182:0:5575:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952af3780 going to sleep for 11 seconds 00000100:00100000:2.0:1775764812.809346:0:5575:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967030016:192.168.202.146@tcp:101:ll_cfg_requeue.0 00010000:00010000:2.0:1775764812.809378:0:5575:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.202.146@tcp lock: ffff8f99821cc600/0x36f9bdc9e421c872 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x2aac6a50e5fb490 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764812.809391:0:5575:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.202.146@tcp lock: ffff8f99821cc600/0x36f9bdc9e421c872 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x2aac6a50e5fb490 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764812.809479:0:5575:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967030144:192.168.202.146@tcp:256:ll_cfg_requeue.0 00000100:00100000:2.0:1775764812.809582:0:5575:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99435609c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764812.813956:0:5575:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967030144:192.168.202.146@tcp:256:ll_cfg_requeue.0 10000000:01000000:2.0:1775764812.814015:0:5575:0:(mgc_request.c:1896:mgc_process_log()) MGC192.168.202.146@tcp: configuration from log 'lustre-cliir' failed (-2). 00010000:00010000:2.0:1775764812.814028:0:5575:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.146@tcp lock: ffff8f99821cc600/0x36f9bdc9e421c872 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x2aac6a50e5fb490 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764812.814040:0:5575:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.202.146@tcp lock: ffff8f99821cc600/0x36f9bdc9e421c872 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x2aac6a50e5fb490 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000400:2.0:1775764812.814046:0:5575:0:(mgc_request.c:1917:mgc_process_log()) MGC192.168.202.146@tcp: IR log lustre-cliir failed, not fatal: rc = -2 10000000:01000000:2.0:1775764812.820872:0:5575:0:(mgc_request.c:611:do_requeue()) updating log lustre-client 10000000:01000000:2.0:1775764812.820877:0:5575:0:(mgc_request.c:1819:mgc_process_log()) Process log lustre-client-ffff8f99523ee800 from 42 10000000:01000000:2.0:1775764812.820881:0:5575:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-client (res 0x65727473756c) 00010000:00010000:2.0:1775764812.820923:0:5575:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.146@tcp lock: ffff8f99706eb600/0x36f9bdc9e421c879 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764812.820931:0:5575:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.146@tcp lock: ffff8f99706eb600/0x36f9bdc9e421c879 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764812.820940:0:5575:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.146@tcp lock: ffff8f99706eb600/0x36f9bdc9e421c879 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764812.820962:0:5575:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967030272:192.168.202.146@tcp:101:ll_cfg_requeue.0 00000100:00100000:2.0:1775764812.821045:0:5575:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99435609c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764812.823881:0:5575:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967030272:192.168.202.146@tcp:101:ll_cfg_requeue.0 00010000:00010000:2.0:1775764812.823906:0:5575:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.202.146@tcp lock: ffff8f99706eb600/0x36f9bdc9e421c879 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x2aac6a50e5fb497 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764812.823915:0:5575:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.202.146@tcp lock: ffff8f99706eb600/0x36f9bdc9e421c879 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x2aac6a50e5fb497 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764812.823978:0:5575:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967030400:192.168.202.146@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1775764812.824042:0:5575:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99435609c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764812.843407:0:5575:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967030400:192.168.202.146@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1775764812.843475:0:5575:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967030528:192.168.202.146@tcp:503:ll_cfg_requeue.0 00000100:00100000:2.0:1775764812.843588:0:5575:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99435609c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764812.846923:0:5575:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967030528:192.168.202.146@tcp:503:ll_cfg_requeue.0 00000040:00080000:1.0:1775764812.848091:0:20720:0:(llog.c:857:llog_process_thread()) stop processing plain [0xa:0x3:0x0] index 64768 count 42 00000020:01000000:2.0:1775764812.848406:0:5575:0:(obd_config.c:2143:class_config_parse_llog()) Processed log lustre-client gen 42-41 (rc=0) 10000000:01000000:2.0:1775764812.848429:0:5575:0:(mgc_request.c:1896:mgc_process_log()) MGC192.168.202.146@tcp: configuration from log 'lustre-client' succeeded (0). 00010000:00010000:2.0:1775764812.848441:0:5575:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.146@tcp lock: ffff8f99706eb600/0x36f9bdc9e421c879 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x2aac6a50e5fb497 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764812.848453:0:5575:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.202.146@tcp lock: ffff8f99706eb600/0x36f9bdc9e421c879 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x2aac6a50e5fb497 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1775764812.848463:0:5575:0:(mgc_request.c:611:do_requeue()) updating log params 10000000:01000000:2.0:1775764812.848466:0:5575:0:(mgc_request.c:1819:mgc_process_log()) Process log params-ffff8f99523ee800 from 7 10000000:01000000:2.0:1775764812.848469:0:5575:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for params (res 0x736d61726170) 00010000:00010000:2.0:1775764812.848511:0:5575:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.146@tcp lock: ffff8f99706eac00/0x36f9bdc9e421c880 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764812.848519:0:5575:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.146@tcp lock: ffff8f99706eac00/0x36f9bdc9e421c880 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764812.848531:0:5575:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.146@tcp lock: ffff8f99706eac00/0x36f9bdc9e421c880 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764812.848565:0:5575:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967030656:192.168.202.146@tcp:101:ll_cfg_requeue.0 00000100:00100000:2.0:1775764812.848713:0:5575:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99435609c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764812.852051:0:5575:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967030656:192.168.202.146@tcp:101:ll_cfg_requeue.0 00010000:00010000:2.0:1775764812.852080:0:5575:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.202.146@tcp lock: ffff8f99706eac00/0x36f9bdc9e421c880 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x2aac6a50e5fb49e expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764812.852092:0:5575:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.202.146@tcp lock: ffff8f99706eac00/0x36f9bdc9e421c880 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x2aac6a50e5fb49e expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764812.852174:0:5575:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967030784:192.168.202.146@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1775764812.852294:0:5575:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99435609c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764812.858969:0:5575:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967030784:192.168.202.146@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1775764812.859222:0:5575:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967030912:192.168.202.146@tcp:503:ll_cfg_requeue.0 00000100:00100000:2.0:1775764812.859362:0:5575:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99435609c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764812.867123:0:5575:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967030912:192.168.202.146@tcp:503:ll_cfg_requeue.0 00000040:00080000:0.0:1775764812.868439:0:20721:0:(llog.c:857:llog_process_thread()) stop processing plain [0xa:0x2:0x0] index 64768 count 7 00000020:01000000:2.0:1775764812.868615:0:5575:0:(obd_config.c:2143:class_config_parse_llog()) Processed log params gen 7-6 (rc=0) 10000000:01000000:2.0:1775764812.868637:0:5575:0:(mgc_request.c:1896:mgc_process_log()) MGC192.168.202.146@tcp: configuration from log 'params' succeeded (0). 00010000:00010000:2.0:1775764812.868647:0:5575:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.146@tcp lock: ffff8f99706eac00/0x36f9bdc9e421c880 lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x2aac6a50e5fb49e expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764812.868659:0:5575:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.202.146@tcp lock: ffff8f99706eac00/0x36f9bdc9e421c880 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x2aac6a50e5fb49e expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1775764812.868669:0:5575:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:01000000:2.0:1775764812.868671:0:5575:0:(mgc_request.c:1819:mgc_process_log()) Process log lustre-sptlrpc-ffff8f9942a1b380 from 1 10000000:01000000:2.0:1775764812.868674:0:5575:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00010000:00010000:2.0:1775764812.868711:0:5575:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.146@tcp lock: ffff8f99706eba00/0x36f9bdc9e421c887 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764812.868723:0:5575:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.146@tcp lock: ffff8f99706eba00/0x36f9bdc9e421c887 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764812.868733:0:5575:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.146@tcp lock: ffff8f99706eba00/0x36f9bdc9e421c887 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764812.868786:0:5575:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967031040:192.168.202.146@tcp:101:ll_cfg_requeue.0 00000100:00100000:2.0:1775764812.868870:0:5575:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99435609c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764812.875097:0:5575:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967031040:192.168.202.146@tcp:101:ll_cfg_requeue.0 00010000:00010000:2.0:1775764812.875140:0:5575:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.202.146@tcp lock: ffff8f99706eba00/0x36f9bdc9e421c887 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x2aac6a50e5fb4a5 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764812.875153:0:5575:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.202.146@tcp lock: ffff8f99706eba00/0x36f9bdc9e421c887 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x2aac6a50e5fb4a5 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764812.875241:0:5575:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967031168:192.168.202.146@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1775764812.875326:0:5575:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99435609c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764812.881393:0:5575:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967031168:192.168.202.146@tcp:501:ll_cfg_requeue.0 10000000:01000000:2.0:1775764812.881441:0:5575:0:(mgc_request.c:1896:mgc_process_log()) MGC192.168.202.146@tcp: configuration from log 'lustre-sptlrpc' failed (-2). 00010000:00010000:2.0:1775764812.881451:0:5575:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.146@tcp lock: ffff8f99706eba00/0x36f9bdc9e421c887 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x2aac6a50e5fb4a5 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764812.881462:0:5575:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.202.146@tcp lock: ffff8f99706eba00/0x36f9bdc9e421c887 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x2aac6a50e5fb4a5 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764813.382062:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764814.408661:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764815.436840:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764816.454136:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775764816.582093:0:20320:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775764816.582108:0:20320:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:0.0:1775764816.582143:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967031424:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764816.582264:0:2380:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:0.0:1775764816.582271:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2380:1862022967031296:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764816.582297:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967031552:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764816.590080:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967031424:192.168.202.146@tcp:400:kworker.0 00000100:00100000:1.0:1775764816.599981:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2380:1862022967031296:192.168.202.146@tcp:400:kworker.0 00000100:00100000:1.0:1775764816.600007:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967031552:192.168.202.146@tcp:400:kworker.0 00000100:00000400:3.0:1775764816.710021:0:2381:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764800/real 1775764800] req@ffff8f9970edf800 x1862022967028096/t0(0) o400->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764816 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764816.767890:0:2381:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8f9970edf800 x1862022967028096/t0(0) o400->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764816 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775764816.767905:0:2381:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff8f9952a6b000 already not connected (conn 2, was 1): CONNECTING 00000100:00100000:3.0:1775764816.767926:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967028096:192.168.202.146@tcp:400:kworker.0 00010000:00010000:0.0:1775764817.479348:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764818.502099:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764819.531560:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764819.568599:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764819.568779:0:11131:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978404480 00000100:00100000:0.0:1775764819.568791:0:11131:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978404480 00000100:00100000:0.0:1775764819.568897:0:11131:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24225 00000100:00100000:0.0:1775764819.568919:0:11131:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e6a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:884ae1b0-72f8-4a88-a129-db99cded7efc+3:17494:x1862022978404480:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764819.568949:0:11131:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.202.146@tcp lock: ffff8f99821cc600/0x36f9bdc9e421c872 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0x2aac6a50e5fb490 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764819.569042:0:11131:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764819.569049:0:11131:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e6a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:884ae1b0-72f8-4a88-a129-db99cded7efc+3:17494:x1862022978404480:12345-192.168.202.146@tcp:104: Request processed in 133us (462us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764819.569057:0:11131:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24225 00010000:00010000:3.0:1775764819.570452:0:19800:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1775764819.570493:0:19800:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.146@tcp lock: ffff8f99821cc600/0x36f9bdc9e421c872 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0x2aac6a50e5fb490 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764819.570503:0:19800:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99821cc600 already unused, calling callback (ffffffffc1393c20) 10000000:00010000:3.0:1775764819.570507:0:19800:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.146@tcp lock: ffff8f99821cc600/0x36f9bdc9e421c872 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1400400000000 nid: local remote: 0x2aac6a50e5fb490 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764819.570535:0:19800:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.146@tcp lock: ffff8f99821cc600/0x36f9bdc9e421c872 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1408400000000 nid: local remote: 0x2aac6a50e5fb490 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:3.0:1775764819.570543:0:19800:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.146@tcp lock: ffff8f99821cc600/0x36f9bdc9e421c872 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1409400000000 nid: local remote: 0x2aac6a50e5fb490 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1775764819.570548:0:19800:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 00010000:00010000:3.0:1775764819.570614:0:19800:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: MGC192.168.202.146@tcp lock: ffff8f99821cc600/0x36f9bdc9e421c872 lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x2aac6a50e5fb490 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764819.570620:0:19800:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764819.570646:0:19800:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.146@tcp lock: ffff8f99821cc600/0x36f9bdc9e421c872 lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x2aac6a50e5fb490 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764819.570652:0:19800:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.146@tcp lock: ffff8f99821cc600/0x36f9bdc9e421c872 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x2aac6a50e5fb490 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764819.570670:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764819.570671:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764819.570673:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764819.570693:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2382:1862022967031680:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764819.584907:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2382:1862022967031680:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1775764820.551694:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:01000000:2.0:1775764821.061882:0:5575:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:01000000:2.0:1775764821.061891:0:5575:0:(mgc_request.c:1819:mgc_process_log()) Process log lustre-cliir-ffff8f99523ee800 from 6 10000000:01000000:2.0:1775764821.061897:0:5575:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00010000:00010000:2.0:1775764821.061959:0:5575:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.146@tcp lock: ffff8f99706ea000/0x36f9bdc9e421c88e lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764821.061973:0:5575:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.146@tcp lock: ffff8f99706ea000/0x36f9bdc9e421c88e lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764821.061985:0:5575:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.146@tcp lock: ffff8f99706ea000/0x36f9bdc9e421c88e lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764821.062020:0:5575:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967031808:192.168.202.146@tcp:101:ll_cfg_requeue.0 00000100:00100000:2.0:1775764821.062310:0:5575:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99435609c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764821.074247:0:5575:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967031808:192.168.202.146@tcp:101:ll_cfg_requeue.0 00010000:00010000:2.0:1775764821.074279:0:5575:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.202.146@tcp lock: ffff8f99706ea000/0x36f9bdc9e421c88e lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x2aac6a50e5fb5a8 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764821.074292:0:5575:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.202.146@tcp lock: ffff8f99706ea000/0x36f9bdc9e421c88e lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x2aac6a50e5fb5a8 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764821.074388:0:5575:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967031936:192.168.202.146@tcp:256:ll_cfg_requeue.0 00000100:00100000:2.0:1775764821.074550:0:5575:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99435609c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764821.090239:0:5575:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967031936:192.168.202.146@tcp:256:ll_cfg_requeue.0 00000020:00000080:2.0:1775764821.090295:0:5575:0:(obd_config.c:1468:class_process_config()) processing cmd: cf00f 00000020:00000080:2.0:1775764821.090439:0:5575:0:(obd_config.c:1468:class_process_config()) processing cmd: cf00f 00000020:00000080:2.0:1775764821.090474:0:5575:0:(obd_config.c:1468:class_process_config()) processing cmd: cf00f 10000000:01000000:2.0:1775764821.090539:0:5575:0:(mgc_request.c:1896:mgc_process_log()) MGC192.168.202.146@tcp: configuration from log 'lustre-cliir' succeeded (0). 00010000:00010000:2.0:1775764821.090550:0:5575:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.146@tcp lock: ffff8f99706ea000/0x36f9bdc9e421c88e lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x2aac6a50e5fb5a8 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764821.090561:0:5575:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.202.146@tcp lock: ffff8f99706ea000/0x36f9bdc9e421c88e lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x2aac6a50e5fb5a8 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00080000:1.0:1775764821.197575:0:20738:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff8f9952a6b000 already not connected (conn 2, was 0): CONNECTING 00010000:00080000:1.0:1775764821.197615:0:20738:0:(ldlm_lib.c:85:import_set_conn()) imp ffff8f9952a6b000@lustre-MDT0000-mdc-ffff8f99523ee800: found existing conn 192.168.202.146@tcp, moved to head 00010000:00010000:0.0:1775764821.575795:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:3.0:1775764821.637862:0:2382:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764805/real 1775764805] req@ffff8f99703d1500 x1862022967028608/t0(0) o400->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764821 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764821.662127:0:2382:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8f99703d1500 x1862022967028608/t0(0) o400->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764821 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775764821.662201:0:2382:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff8f9952a6b000 already not connected (conn 2, was 1): CONNECTING 00000100:00100000:3.0:1775764821.662223:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967028608:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764821.702016:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764811/real 1775764811] req@ffff8f99703d2d80 x1862022967029632/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764821 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764821.702051:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2d80 x1862022967029632/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764821 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764821.702071:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775764821.702094:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775764821.702099:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967029632:192.168.202.146@tcp:38:kworker.0 00000100:00080000:3.0:1775764821.702224:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764821.702231:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775764821.702234:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764821.702241:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 0 00000100:00080000:3.0:1775764821.702256:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775764821.702268:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764821.702341:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967032192/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764821.702410:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967032320:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764821.702515:0:2382:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:3.0:1775764821.702522:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2382:1862022967032064:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764821.702570:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967032448:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764821.702901:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967032192:192.168.202.146@tcp:38:kworker.0 00010000:00080000:1.0:1775764821.720377:0:2378:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff8f9970ede300 x1862022967032192/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/416 e 0 to 0 dl 1775764831 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764821.720395:0:2378:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8f99523ee800: committing for last_committed 4294997605 gen 1 00000100:00100000:1.0:1775764821.720599:0:2378:0:(client.c:3126:ptlrpc_free_committed()) @@@ stopping search req@ffff8f9970ede680 x1862022967025152/t4294997608(4294997608) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 488/456 e 0 to 0 dl 1775764805 ref 1 fl Complete:RQU/204/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764821.720617:0:2378:0:(import.c:1130:ptlrpc_connect_interpret()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to target with instance 2 00000100:00080000:1.0:1775764821.720638:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to REPLAY 00000100:00080000:1.0:1775764821.720645:0:2378:0:(import.c:1694:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:1.0:1775764821.720648:0:2378:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8f99523ee800: committing for last_committed 4294997605 gen 1 00000100:00100000:1.0:1775764821.720652:0:2378:0:(client.c:3126:ptlrpc_free_committed()) @@@ stopping search req@ffff8f9970ede680 x1862022967025152/t4294997608(4294997608) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 488/456 e 0 to 0 dl 1775764805 ref 1 fl Complete:RQU/204/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764821.720660:0:2378:0:(recover.c:53:ptlrpc_replay_next()) import ffff8f9952a6b000 from lustre-MDT0000_UUID committed 4294997605 last 0 00000100:00080000:1.0:1775764821.720668:0:2378:0:(client.c:3524:ptlrpc_replay_req()) @@@ REPLAY req@ffff8f9970ede680 x1862022967025152/t4294997608(4294997608) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 488/456 e 0 to 0 dl 1775764805 ref 1 fl New:RQU/204/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764821.720681:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967032192:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775764821.720717:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967025152:192.168.202.146@tcp:36:dd.0 00000100:00100000:3.0:1775764821.726507:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2382:1862022967032064:192.168.202.146@tcp:400:kworker.0 00010000:00010000:3.0:1775764821.726555:0:2382:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 0/0 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1775764821.726563:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967032448:192.168.202.146@tcp:400:kworker.0 00010000:00010000:3.0:1775764821.726574:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1775764821.726590:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764821.726592:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764821.726593:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764821.726687:0:2382:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 0/0 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1775764821.726694:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967032320:192.168.202.146@tcp:400:kworker.0 00010000:00010000:1.0:1775764821.727421:0:11138:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764821.727433:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764821.727434:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764821.727436:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775764821.772728:0:2378:0:(client.c:3409:ptlrpc_replay_interpret()) @@@ got reply req@ffff8f9970ede680 x1862022967025152/t4294997608(4294997608) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 488/456 e 0 to 0 dl 1775764837 ref 2 fl Interpret:RQU/204/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764821.772793:0:2378:0:(import.c:1694:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:0.0:1775764821.772796:0:2378:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8f99523ee800: committing for last_committed 4294997605 gen 1 00000100:00100000:0.0:1775764821.772801:0:2378:0:(client.c:3126:ptlrpc_free_committed()) @@@ stopping search req@ffff8f9970ede680 x1862022967025152/t4294997608(4294997608) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 488/456 e 0 to 0 dl 1775764837 ref 2 fl Interpret:RQU/204/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764821.772846:0:2378:0:(recover.c:53:ptlrpc_replay_next()) import ffff8f9952a6b000 from lustre-MDT0000_UUID committed 4294997605 last 4294997608 00000100:00080000:0.0:1775764821.772917:0:2378:0:(client.c:3524:ptlrpc_replay_req()) @@@ REPLAY req@ffff8f9970edce00 x1862022967025536/t4294997609(4294997609) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 616/584 e 0 to 0 dl 1775764805 ref 1 fl New:RQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1775764821.772931:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967025152:192.168.202.146@tcp:36:dd.0 00000100:00100000:0.0:1775764821.772953:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967025536:192.168.202.146@tcp:101:dd.0 00000100:00080000:0.0:1775764821.782891:0:2378:0:(client.c:3409:ptlrpc_replay_interpret()) @@@ got reply req@ffff8f9970edce00 x1862022967025536/t4294997609(4294997609) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 616/584 e 0 to 0 dl 1775764837 ref 2 fl Interpret:RQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000100:00080000:0.0:1775764821.782905:0:2378:0:(import.c:1694:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:0.0:1775764821.782908:0:2378:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8f99523ee800: committing for last_committed 4294997605 gen 1 00000100:00100000:0.0:1775764821.782924:0:2378:0:(client.c:3126:ptlrpc_free_committed()) @@@ stopping search req@ffff8f9970ede680 x1862022967025152/t4294997608(4294997608) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 488/456 e 0 to 0 dl 1775764837 ref 1 fl Complete:RQU/204/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764821.782932:0:2378:0:(recover.c:53:ptlrpc_replay_next()) import ffff8f9952a6b000 from lustre-MDT0000_UUID committed 4294997605 last 4294997609 00000100:00080000:0.0:1775764821.782936:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from REPLAY to REPLAY_LOCKS 00000100:00100000:1.0:1775764821.783253:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967025536:192.168.202.146@tcp:101:dd.0 00010000:00080000:0.0:1775764821.783299:0:20775:0:(ldlm_request.c:2960:ldlm_lock_replay_thread()) lock replay thread lustre-MDT0000-mdc-ffff8f99523ee800 to lustre-MDT0000_UUID@192.168.202.146@tcp 00010000:00010000:0.0:1775764821.783303:0:20775:0:(ldlm_request.c:2876:ldlm_cancel_unused_locks_for_replay()) Dropping as many unused locks as possible before replay for namespace lustre-MDT0000-mdc-ffff8f99523ee800 (3) 00010000:00010000:0.0:1775764821.783342:0:20775:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c83a lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000000 nid: local remote: 0x2aac6a50e5fb3e1 expref: -99 pid: 20529 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1775764821.783456:0:20775:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0xed9:0x0]: losing layout lock 00010000:00010000:0.0:1775764821.783491:0:20775:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c84f lrc: 0/0,0 mode: --/CR res: [0x200000401:0xed9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000000 nid: local remote: 0x2aac6a50e5fb420 expref: -99 pid: 20529 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764821.783518:0:20775:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c7f4 lrc: 0/0,0 mode: --/PR res: [0x200000401:0xed8:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000000 nid: local remote: 0x2aac6a50e5fb2d0 expref: -99 pid: 20471 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764821.783539:0:20775:0:(ldlm_request.c:2889:ldlm_cancel_unused_locks_for_replay()) Canceled 0 unused locks from namespace lustre-MDT0000-mdc-ffff8f99523ee800 00000100:00080000:0.0:1775764821.784187:0:20775:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from REPLAY_LOCKS to REPLAY_WAIT 00000100:00100000:1.0:1775764821.784370:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967032576:192.168.202.146@tcp:400:ldlm_lock_repla.0 00000100:00080000:0.0:1775764821.854293:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from REPLAY_WAIT to RECOVER 00000100:00080000:0.0:1775764821.854311:0:2378:0:(client.c:3204:ptlrpc_resend_req()) @@@ going to resend req@ffff8f9970a74700 x1862022967032576/t0(0) o400->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764837 ref 1 fl Interpret:RQU/2c0/0 rc 0/0 job:'ldlm_lock_repla.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764821.854322:0:2378:0:(client.c:3213:ptlrpc_resend_req()) @@@ it has reply, so skip it req@ffff8f9970a74700 x1862022967032576/t0(0) o400->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764837 ref 1 fl Interpret:RQU/2c0/0 rc 0/0 job:'ldlm_lock_repla.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764821.854330:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from RECOVER to FULL 00000100:02000000:1.0:1775764821.855704:0:2378:0:(import.c:1737:ptlrpc_import_recovery_state_machine()) lustre-MDT0000-mdc-ffff8f99523ee800: Connection restored to 192.168.202.146@tcp (at 192.168.202.146@tcp) 00000100:00100000:1.0:1775764821.859488:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967032576:192.168.202.146@tcp:400:ldlm_lock_repla.0 00010000:00010000:1.0:1775764822.597975:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764823.621968:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764824.647082:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764825.673780:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764826.710348:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764826.822927:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967032832:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764826.823064:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967033088:192.168.202.146@tcp:400:kworker.0 00000100:00100000:2.0:1775764826.823230:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2382:1862022967032704:192.168.202.146@tcp:400:kworker.0 00000100:00100000:2.0:1775764826.824391:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967032960:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764826.832794:0:2381:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8f99523ee800: committing for last_committed 4294997609 gen 1 00000100:00100000:3.0:1775764826.832870:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967032832:192.168.202.146@tcp:400:kworker.0 00000100:00100000:2.0:1775764826.834840:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967032960:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764826.842850:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967033088:192.168.202.146@tcp:400:kworker.0 00000100:00100000:2.0:1775764826.848860:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2382:1862022967032704:192.168.202.146@tcp:400:kworker.0 00010000:00010000:1.0:1775764827.718175:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:2.0:1775764827.847733:0:2381:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764810/real 1775764810] req@ffff8f99703d0e00 x1862022967029120/t0(0) o400->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764826 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764827.887647:0:2381:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8f99703d0e00 x1862022967029120/t0(0) o400->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764826 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764827.887681:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967029120:192.168.202.146@tcp:400:kworker.0 00010000:00010000:1.0:1775764828.743715:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:3.0:1775764829.491219:0:20829:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8f9952d121c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764829.491237:0:20829:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:3.0:1775764829.491262:0:20829:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764829.491331:0:20829:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 20829 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764829.491342:0:20829:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 20829 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764829.491354:0:20829:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 20829 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764829.491381:0:20829:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:20829:1862022967033216:192.168.202.146@tcp:101:lfs.0 00000100:00100000:3.0:1775764829.491539:0:20829:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f997130d000 going to sleep for 11 seconds 00000100:00100000:3.0:1775764829.495587:0:20829:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:20829:1862022967033216:192.168.202.146@tcp:101:lfs.0 00010000:00010000:3.0:1775764829.495617:0:20829:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764829.495679:0:20829:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764829.495691:0:20829:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764829.495703:0:20829:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764829.495716:0:20829:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764829.495724:0:20829:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764829.495956:0:20829:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764829.495968:0:20829:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970edfb80 x1862022967033216/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764845 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1775764829.495982:0:20829:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764829.495995:0:20829:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1775764829.496002:0:20829:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1775764829.496032:0:20829:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 8704 00000080:00010000:3.0:1775764829.496055:0:20829:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c895 00000080:00010000:3.0:1775764829.496062:0:20829:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:3.0:1775764829.496065:0:20829:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c895 from it ffffafbac0d4bcb8 00010000:00010000:3.0:1775764829.496068:0:20829:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764829.496086:0:20829:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:3.0:1775764829.498495:0:20829:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8f9952d121c8), request_mask 0, flags 0, foreign 0 00000080:00200000:3.0:1775764829.498508:0:20829:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:3.0:1775764829.498533:0:20829:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764829.498555:0:20829:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764829.498573:0:20829:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764829.498581:0:20829:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764829.498590:0:20829:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:3.0:1775764829.498593:0:20829:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c895 00000080:00010000:3.0:1775764829.498601:0:20829:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c895 from it ffffafbac0d4bcb8 00010000:00010000:3.0:1775764829.498605:0:20829:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764829.498629:0:20829:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764829.498901:0:20829:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:3.0:1775764829.498910:0:20829:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764829.498917:0:20829:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764829.498925:0:20829:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764829.498931:0:20829:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764829.498935:0:20829:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:3.0:1775764829.498937:0:20829:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c895 00000080:00010000:3.0:1775764829.498941:0:20829:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c895 from it ffffafbac0d4bbb0 00010000:00010000:3.0:1775764829.498943:0:20829:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764829.498949:0:20829:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764829.498957:0:20829:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 44 00000080:00200008:3.0:1775764829.498966:0:20829:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8f9952d121c8), flags 100000 00000080:00200080:3.0:1775764829.499097:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.499115:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00800000:00000004:3.0:1775764829.499122:0:20829:0:(obd_class.h:1092:obd_statfs()) lustre-MDT0000-mdc-ffff8f99523ee800: age 1412, max_age 1455 00000100:00100000:3.0:1775764829.499172:0:20829:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:20829:1862022967033344:192.168.202.146@tcp:41:lfs.0 00000100:00100000:3.0:1775764829.499286:0:20829:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f997130d000 going to sleep for 11 seconds 00000100:00100000:3.0:1775764829.507019:0:20829:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:20829:1862022967033344:192.168.202.146@tcp:41:lfs.0 00800000:00000004:3.0:1775764829.507093:0:20829:0:(obd_class.h:1118:obd_statfs()) lustre-MDT0000-mdc-ffff8f99523ee800: update ffff8f9952360450 cache blocks 17235/17269 objects 528850/529200 00000080:00200080:3.0:1775764829.507186:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507204:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507213:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507217:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507221:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507225:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507229:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507232:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507235:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507239:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507242:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507245:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507249:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507251:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507255:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507258:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507262:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507265:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507268:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507271:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507275:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507278:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507282:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507285:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507307:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507311:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507315:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507319:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507323:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507327:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507330:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507334:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507337:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507341:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507346:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507349:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507352:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507356:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507360:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507364:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507367:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507371:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507375:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507378:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507382:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507385:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507388:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507391:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507395:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507413:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507418:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507422:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507425:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507428:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507432:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507435:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507439:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507443:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507446:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507450:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507454:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507457:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507461:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507464:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507469:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507472:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507475:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507479:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507482:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507484:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507487:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507491:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507494:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507497:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507517:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507521:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507525:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507529:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507532:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507534:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507538:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507541:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507544:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507547:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507550:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507552:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507556:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507559:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507562:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507565:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507569:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507572:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507575:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507578:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507582:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507584:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507588:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507591:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507594:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507612:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507617:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507621:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507624:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507628:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507631:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507634:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507638:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507641:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507644:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507646:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507650:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507652:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507656:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507659:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507662:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507665:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507668:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507672:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507675:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507678:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507681:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507684:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507687:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507690:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507707:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507712:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507715:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00800000:00000080:3.0:1775764829.507718:0:20829:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.507722:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.507769:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00020000:00000004:3.0:1775764829.507778:0:20829:0:(obd_class.h:1092:obd_statfs()) lustre-OST0000-osc-ffff8f99523ee800: age 1412, max_age 1455 00000100:00100000:3.0:1775764829.507815:0:20829:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:20829:1862022967033472:192.168.202.146@tcp:13:lfs.0 00000100:00100000:3.0:1775764829.507892:0:20829:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f997130d000 going to sleep for 11 seconds 00000100:00100000:3.0:1775764829.517220:0:20829:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:20829:1862022967033472:192.168.202.146@tcp:13:lfs.0 00020000:00000004:3.0:1775764829.517250:0:20829:0:(obd_class.h:1118:obd_statfs()) lustre-OST0000-osc-ffff8f99523ee800: update ffff8f99523637d0 cache blocks 3677/3683 objects 117728/119224 00000080:00200080:3.0:1775764829.517323:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.517342:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00020000:00000004:3.0:1775764829.517350:0:20829:0:(obd_class.h:1092:obd_statfs()) lustre-OST0001-osc-ffff8f99523ee800: age 1412, max_age 1455 00000100:00100000:3.0:1775764829.517390:0:20829:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:20829:1862022967033600:192.168.202.146@tcp:13:lfs.0 00000100:00100000:3.0:1775764829.517480:0:20829:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f997130d000 going to sleep for 11 seconds 00000100:00100000:3.0:1775764829.526595:0:20829:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:20829:1862022967033600:192.168.202.146@tcp:13:lfs.0 00020000:00000004:3.0:1775764829.526624:0:20829:0:(obd_class.h:1118:obd_statfs()) lustre-OST0001-osc-ffff8f99523ee800: update ffff8f9951ec8450 cache blocks 3655/3683 objects 117760/119255 00000080:00200080:3.0:1775764829.526702:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.526724:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.526782:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.526789:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.526793:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.526797:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.526838:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.526842:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.526846:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.526849:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.526853:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.526856:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.526860:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.526863:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.526867:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.526870:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.526875:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.526877:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.526881:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.526884:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.526888:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.526891:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.526895:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.526898:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.526902:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.526905:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.526909:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.526913:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.526917:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.526920:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.526924:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.526927:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.526931:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.526934:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.526938:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.526941:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.526944:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.526948:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.526951:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.526955:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.526959:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.526962:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.526965:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.526968:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.526972:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.526975:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.526978:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.526980:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.526983:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.526986:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.526990:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.526993:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.526996:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.526999:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.527003:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.527006:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.527026:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.527031:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.527035:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.527038:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.527042:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.527045:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.527048:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.527051:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.527055:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.527058:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.527062:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.527065:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.527068:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.527071:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.527087:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.527092:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.527099:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.527102:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.527106:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.527109:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.527112:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.527115:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.527119:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.527122:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.527126:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.527143:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.527148:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.527152:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.527156:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.527159:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.527162:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.527167:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.527171:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.527175:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.527180:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.527183:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.527186:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.527190:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.527193:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.527197:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.527200:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.527203:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.527207:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.527210:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.527214:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.527217:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.527220:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.527224:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.527228:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.527231:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.527248:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.527252:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.527257:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.527260:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.527263:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.527267:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.527271:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.527275:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.527278:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.527282:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.527285:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.527288:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.527292:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.527294:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.527298:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.527301:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.527304:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.527307:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200080:3.0:1775764829.527311:0:20829:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7ffd3129cda0 00020000:00000080:3.0:1775764829.527314:0:20829:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966931c00 uarg=0000000000000000 00000080:00200008:3.0:1775764829.527330:0:20829:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8f9952d121c8), flags 100000 00010000:00010000:3.0:1775764829.766204:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764830.795764:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764831.819004:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764831.942045:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2380:1862022967033728:192.168.202.146@tcp:400:kworker.0 00000100:00100000:1.0:1775764831.942163:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967033984:192.168.202.146@tcp:400:kworker.0 00000100:00100000:1.0:1775764831.942197:0:2380:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1775764831.942203:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967033856:192.168.202.146@tcp:400:kworker.0 00000100:00100000:1.0:1775764831.942224:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967034112:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764831.952360:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967033856:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764831.952389:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967034112:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764831.952400:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2380:1862022967033728:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764831.952409:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967033984:192.168.202.146@tcp:400:kworker.0 00010000:00010000:1.0:1775764832.838541:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:3.0:1775764833.497599:0:21079:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg246-client.virtnet: executing wait_import_state_mount (FULL|IDLE) mdc.lustre-MDT0000-mdc-*.mds_server_uuid 00010000:00010000:1.0:1775764833.862413:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764834.886788:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:2.0:1775764835.397018:0:21234:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: mdc.lustre-MDT0000-mdc-*.mds_server_uuid in FULL state after 0 sec 00010000:00010000:3.0:1775764835.909898:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764836.933953:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764837.064383:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2380:1862022967034240:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764837.064676:0:2380:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1775764837.064684:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967034368:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764837.065163:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967034496:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764837.065263:0:2380:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1775764837.065270:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967034624:192.168.202.146@tcp:400:kworker.0 00000100:00100000:1.0:1775764837.077954:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967034496:192.168.202.146@tcp:400:kworker.0 00000100:00100000:1.0:1775764837.077973:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2380:1862022967034240:192.168.202.146@tcp:400:kworker.0 00000100:00100000:1.0:1775764837.083164:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967034368:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764837.088988:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967034624:192.168.202.146@tcp:400:kworker.0 00000080:00200008:3.0:1775764837.456933:0:21272:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8f9952d121c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764837.456988:0:21272:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:3.0:1775764837.457018:0:21272:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764837.457041:0:21272:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764837.457057:0:21272:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764837.457064:0:21272:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764837.457072:0:21272:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:3.0:1775764837.457076:0:21272:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c895 00000080:00010000:3.0:1775764837.457083:0:21272:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c895 from it ffffafbac0e33cb8 00010000:00010000:3.0:1775764837.457087:0:21272:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764837.457095:0:21272:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200008:3.0:1775764837.458883:0:21272:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8f9952d121c8), request_mask 0, flags 0, foreign 0 00000080:00200000:3.0:1775764837.458892:0:21272:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:3.0:1775764837.458910:0:21272:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764837.458928:0:21272:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1775764837.458941:0:21272:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1775764837.458949:0:21272:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764837.458978:0:21272:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:3.0:1775764837.458981:0:21272:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c895 00000080:00010000:3.0:1775764837.458987:0:21272:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c895 from it ffffafbac0e33cb8 00010000:00010000:3.0:1775764837.458991:0:21272:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1775764837.459000:0:21272:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1775764837.459171:0:21272:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:3.0:1775764837.459180:0:21272:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764837.459186:0:21272:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1775764837.459195:0:21272:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1775764837.459201:0:21272:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764837.459205:0:21272:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:3.0:1775764837.459207:0:21272:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c895 00000080:00010000:3.0:1775764837.459211:0:21272:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c895 from it ffffafbac0e33bb0 00010000:00010000:3.0:1775764837.459213:0:21272:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1775764837.459221:0:21272:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:3.0:1775764837.459228:0:21272:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 44 00000080:00200008:3.0:1775764837.459250:0:21272:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8f9952d121c8), flags 100000 00000080:00200080:3.0:1775764837.459368:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.459385:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00800000:00000004:3.0:1775764837.460663:0:21272:0:(obd_class.h:1092:obd_statfs()) lustre-MDT0000-mdc-ffff8f99523ee800: age 1456, max_age 1463 00000100:00100000:3.0:1775764837.460722:0:21272:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21272:1862022967034752:192.168.202.146@tcp:41:lfs.0 00000100:00100000:3.0:1775764837.460940:0:21272:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99601023c0 going to sleep for 11 seconds 00000100:00100000:3.0:1775764837.468637:0:21272:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21272:1862022967034752:192.168.202.146@tcp:41:lfs.0 00800000:00000004:3.0:1775764837.468667:0:21272:0:(obd_class.h:1118:obd_statfs()) lustre-MDT0000-mdc-ffff8f99523ee800: update ffff8f9952360450 cache blocks 17235/17269 objects 528850/529200 00000080:00200080:3.0:1775764837.468783:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.468800:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.468809:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.468813:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.468817:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.468821:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.468824:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.468828:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.468832:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.468835:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.468839:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.468842:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.468846:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.468850:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.468854:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.468857:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.468861:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.468865:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.468868:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.468872:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.468875:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.468878:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.468882:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.468885:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.468888:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.468891:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.468894:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.468897:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.468900:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.468903:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.468906:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.468909:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.468913:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.468915:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.468919:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.468922:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.468925:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.468927:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.468930:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.468933:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.468936:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.468939:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.468942:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.468945:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.468948:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.468951:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.468954:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.468957:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.468960:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.468963:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.468966:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.468970:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.468973:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.468976:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.468979:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.468982:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.468986:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.468989:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.468992:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.468995:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.468999:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.469013:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.469018:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.469021:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.469025:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.469028:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.469031:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.469034:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.469038:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.469041:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.469044:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.469047:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.469051:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.469054:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.469057:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.469060:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.469064:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.469067:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.469070:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.469073:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.469076:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.469079:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.469083:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.469086:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.469089:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.469092:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.469095:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.469098:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.469101:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.469103:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.469107:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.469110:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.469114:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.469116:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.469120:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.469123:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.469126:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.469129:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.469132:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.469135:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.469138:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.469141:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.469145:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.469148:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.469151:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.469154:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.469157:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.469160:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.469164:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.469167:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.469170:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.469173:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.469177:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.469180:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.469184:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.469187:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.469191:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.469218:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.469223:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.469226:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.469230:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.469233:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.469237:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.469240:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.469243:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.469246:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.469250:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00800000:00000080:3.0:1775764837.469253:0:21272:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.469256:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.469262:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00020000:00000004:3.0:1775764837.469270:0:21272:0:(obd_class.h:1092:obd_statfs()) lustre-OST0000-osc-ffff8f99523ee800: age 1456, max_age 1463 00000100:00100000:3.0:1775764837.469312:0:21272:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21272:1862022967034880:192.168.202.146@tcp:13:lfs.0 00000100:00100000:3.0:1775764837.469381:0:21272:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99601023c0 going to sleep for 11 seconds 00000100:00100000:3.0:1775764837.474861:0:21272:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21272:1862022967034880:192.168.202.146@tcp:13:lfs.0 00020000:00000004:3.0:1775764837.474889:0:21272:0:(obd_class.h:1118:obd_statfs()) lustre-OST0000-osc-ffff8f99523ee800: update ffff8f99523637d0 cache blocks 3677/3683 objects 117728/119224 00000080:00200080:3.0:1775764837.474956:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.474972:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00020000:00000004:3.0:1775764837.474979:0:21272:0:(obd_class.h:1092:obd_statfs()) lustre-OST0001-osc-ffff8f99523ee800: age 1456, max_age 1463 00000100:00100000:3.0:1775764837.475024:0:21272:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21272:1862022967035008:192.168.202.146@tcp:13:lfs.0 00000100:00100000:3.0:1775764837.475090:0:21272:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99601023c0 going to sleep for 11 seconds 00000100:00100000:3.0:1775764837.479065:0:21272:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21272:1862022967035008:192.168.202.146@tcp:13:lfs.0 00020000:00000004:3.0:1775764837.479134:0:21272:0:(obd_class.h:1118:obd_statfs()) lustre-OST0001-osc-ffff8f99523ee800: update ffff8f9951ec8450 cache blocks 3655/3683 objects 117760/119255 00000080:00200080:3.0:1775764837.479210:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479229:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479236:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479240:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479244:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479247:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479252:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479255:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479259:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479263:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479267:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479270:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479274:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479277:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479281:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479284:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479287:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479290:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479294:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479297:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479306:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479309:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479313:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479315:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479341:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479345:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479348:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479351:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479355:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479357:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479361:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479363:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479367:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479369:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479373:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479375:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479379:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479382:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479392:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479405:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479409:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479413:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479416:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479420:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479424:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479427:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479430:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479433:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479437:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479454:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479460:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479463:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479466:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479469:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479472:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479476:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479479:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479482:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479485:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479488:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479492:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479495:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479498:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479501:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479504:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479508:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479510:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479513:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479517:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479520:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479524:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479526:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479530:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479533:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479550:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479554:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479666:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479672:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479676:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479679:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479682:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479685:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479688:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479692:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479696:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479699:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479703:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479706:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479710:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479713:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479717:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479720:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479724:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479727:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479731:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479735:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479767:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479773:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479776:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479793:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479797:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479801:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479804:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479807:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479811:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479814:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479817:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479820:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479824:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479827:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479830:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479833:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479837:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479840:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479843:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479846:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479849:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479853:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479856:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479860:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479864:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479867:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479871:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479874:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200080:3.0:1775764837.479891:0:21272:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=c00866a4 arg=7fffffce34b0 00020000:00000080:3.0:1775764837.479896:0:21272:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=c00866a4 len=592 karg=ffff8f9966932000 uarg=0000000000000000 00000080:00200008:3.0:1775764837.479911:0:21272:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8f9952d121c8), flags 100000 00000080:00200000:2.0:1775764837.516677:0:21290:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:2.0:1775764837.516707:0:21290:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764837.516723:0:21290:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764837.516735:0:21290:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:2.0:1775764837.516777:0:21290:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764837.516786:0:21290:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:2.0:1775764837.516790:0:21290:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c895 00000080:00010000:2.0:1775764837.516798:0:21290:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c895 from it ffffafbac1473bf8 00010000:00010000:2.0:1775764837.516802:0:21290:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764837.516810:0:21290:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:2.0:1775764837.516825:0:21290:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:2.0:1775764837.516853:0:21290:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=d9.sanity-pfl, dir=[0x200000007:0x1:0x0](ffff8f9952d121c8), flags=27 00000080:00200000:2.0:1775764837.516858:0:21290:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=d9.sanity-pfl, dir=[0x200000007:0x1:0x0](ffff8f9952d121c8), intent=0 00000002:00010000:2.0:1775764837.516871:0:21290:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: d9.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: lookup flags 0 00010000:00010000:2.0:1775764837.516917:0:21290:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea200/0x36f9bdc9e421c89c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21290 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764837.516925:0:21290:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea200/0x36f9bdc9e421c89c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21290 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764837.516934:0:21290:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea200/0x36f9bdc9e421c89c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21290 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764837.516957:0:21290:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21290:1862022967035136:192.168.202.146@tcp:101:lfs.0 00000100:00100000:2.0:1775764837.517057:0:21290:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99538df000 going to sleep for 11 seconds 00000100:00100000:2.0:1775764837.530662:0:21290:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21290:1862022967035136:192.168.202.146@tcp:101:lfs.0 00010000:00010000:2.0:1775764837.530699:0:21290:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea200/0x36f9bdc9e421c89c lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf41 expref: -99 pid: 21290 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764837.530722:0:21290:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea200/0x36f9bdc9e421c89c lrc: 4/1,0 mode: --/PR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf41 expref: -99 pid: 21290 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764837.530730:0:21290:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea200/0x36f9bdc9e421c89c lrc: 4/1,0 mode: --/PR res: [0x200000401:0xed8:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf41 expref: -99 pid: 21290 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764837.530759:0:21290:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea200/0x36f9bdc9e421c89c lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xed8:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbf41 expref: -99 pid: 21290 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764837.530772:0:21290:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea200/0x36f9bdc9e421c89c lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xed8:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf41 expref: -99 pid: 21290 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764837.530785:0:21290:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea200/0x36f9bdc9e421c89c lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xed8:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbf41 expref: -99 pid: 21290 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764837.530792:0:21290:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea200/0x36f9bdc9e421c89c lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xed8:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbf41 expref: -99 pid: 21290 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764837.530799:0:21290:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea200/0x36f9bdc9e421c89c lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xed8:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbf41 expref: -99 pid: 21290 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764837.530811:0:21290:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=b, status=0 req@ffff8f99703d0e00 x1862022967035136/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/584 e 0 to 0 dl 1775764853 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764837.530824:0:21290:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea200/0x36f9bdc9e421c89c lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xed8:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf41 expref: -99 pid: 21290 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764837.530833:0:21290:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/3800 (0 0) 00000080:00200000:2.0:1775764837.530840:0:21290:0:(namei.c:943:ll_intent_lock()) intent lock 16 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764837.530862:0:21290:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0xed8:0x0], updating i_size 7680 00000080:00200000:2.0:1775764837.530866:0:21290:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0xed8:0x0](ffff8f9952eb4b08): rc = 0 00000080:00010000:2.0:1775764837.530871:0:21290:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xed8:0x0](ffff8f9952eb4b08) for lock 0x36f9bdc9e421c89c 00000080:00010000:2.0:1775764837.530883:0:21290:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xed8:0x0](ffff8f9952eb4b08) 00000080:00010000:2.0:1775764837.530887:0:21290:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c89c from it ffffafbac1473ac8 00010000:00010000:2.0:1775764837.530890:0:21290:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea200/0x36f9bdc9e421c89c lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xed8:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbf41 expref: -99 pid: 21290 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764837.530898:0:21290:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea200/0x36f9bdc9e421c89c lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xed8:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf41 expref: -99 pid: 21290 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764837.530931:0:21290:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xed8:0x0](ffff8f9952eb4b08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764837.530948:0:21290:0:(namei.c:1399:ll_atomic_open()) VFS Op:name=f9.sanity-pfl, dir=[0x200000401:0xed8:0x0](ffff8f9952eb4b08), file ffff8f9942fd8d80, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1775764837.530954:0:21290:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=f9.sanity-pfl, dir=[0x200000401:0xed8:0x0](ffff8f9952eb4b08), intent=open 00000002:00010000:2.0:1775764837.530973:0:21290:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f9.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000401:0xed8:0x0], intent: open flags 0304001 00010000:00010000:2.0:1775764837.531028:0:21290:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb400/0x36f9bdc9e421c8a3 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21290 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764837.531036:0:21290:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb400/0x36f9bdc9e421c8a3 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21290 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764837.531045:0:21290:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764837.531051:0:21290:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f99703d0e00 x1862022967035392/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 768/131968 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764837.531059:0:21290:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb400/0x36f9bdc9e421c8a3 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21290 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764837.531077:0:21290:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21290:1862022967035392:192.168.202.146@tcp:101:lfs.0 00000100:00100000:2.0:1775764837.531225:0:21290:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99538df000 going to sleep for 11 seconds 00000100:00100000:1.0:1775764837.548295:0:21290:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21290:1862022967035392:192.168.202.146@tcp:101:lfs.0 00010000:00010000:1.0:1775764837.548320:0:21290:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb400/0x36f9bdc9e421c8a3 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21290 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764837.548585:0:21290:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb400/0x36f9bdc9e421c8a3 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21290 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764837.548593:0:21290:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb400/0x36f9bdc9e421c8a3 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21290 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764837.548601:0:21290:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb400/0x36f9bdc9e421c8a3 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21290 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764837.548689:0:21290:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb400/0x36f9bdc9e421c8a3 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21290 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764837.548696:0:21290:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99706eb400 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:1.0:1775764837.548700:0:21290:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb400/0x36f9bdc9e421c8a3 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21290 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764837.548707:0:21290:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb400/0x36f9bdc9e421c8a3 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21290 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764837.548715:0:21290:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb400/0x36f9bdc9e421c8a3 lrc: 5/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21290 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764837.548722:0:21290:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1775764837.548728:0:21290:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb400/0x36f9bdc9e421c8a3 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21290 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764837.548967:0:21290:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb400/0x36f9bdc9e421c8a3 lrc: 0/0,0 mode: --/CR res: [0x200000401:0xed8:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21290 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764837.548985:0:21290:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff8f99703d0e00 x1862022967035392/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 768/608 e 0 to 0 dl 1775764853 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764837.548998:0:21290:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0xed8:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1775764837.549122:0:21290:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764837.549139:0:21290:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764837.549154:0:21290:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764837.549165:0:21290:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1775764837.549172:0:21290:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764837.549177:0:21290:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764837.549181:0:21290:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c895 00000080:00010000:1.0:1775764837.549186:0:21290:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c895 from it ffffafbac1473bf8 00010000:00010000:1.0:1775764837.549188:0:21290:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764837.549195:0:21290:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1775764837.549208:0:21290:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764837.549215:0:21290:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d9.sanity-pfl, flags=267 00000080:00200000:1.0:1775764837.549220:0:21290:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xed8:0x0](ffff8f9952eb4b08), inode mode 41ed mask 44 00000080:00200008:1.0:1775764837.549241:0:21290:0:(file.c:1057:ll_file_open()) START file d9.sanity-pfl:[0x200000401:0xed8:0x0](ffff8f9952eb4b08), flags 304000 00000002:00010000:1.0:1775764837.549254:0:21290:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xed8:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100240304001 00010000:00010000:1.0:1775764837.549295:0:21290:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421c8aa lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21290 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764837.549302:0:21290:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421c8aa lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21290 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764837.549312:0:21290:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764837.549319:0:21290:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e4380 x1862022967035648/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 760/131968 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764837.549560:0:21290:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421c8aa lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21290 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764837.549582:0:21290:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4380 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21290:1862022967035648:192.168.202.146@tcp:101:lfs.0 00000100:00100000:1.0:1775764837.550079:0:21290:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99701d7240 going to sleep for 11 seconds 00000100:00100000:3.0:1775764837.554138:0:21290:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4380 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21290:1862022967035648:192.168.202.146@tcp:101:lfs.0 00010000:00010000:3.0:1775764837.554164:0:21290:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421c8aa lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21290 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764837.554178:0:21290:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421c8aa lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21290 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764837.554185:0:21290:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421c8aa lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21290 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764837.554191:0:21290:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421c8aa lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21290 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764837.554201:0:21290:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421c8aa lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21290 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764837.554207:0:21290:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970641e00 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:3.0:1775764837.554212:0:21290:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421c8aa lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21290 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764837.554219:0:21290:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421c8aa lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21290 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764837.554227:0:21290:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421c8aa lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21290 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764837.554234:0:21290:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1775764837.554240:0:21290:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421c8aa lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21290 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764837.554289:0:21290:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421c8aa lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21290 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764837.554306:0:21290:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8f994f2e4380 x1862022967035648/t8589934593(8589934593) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 760/608 e 0 to 0 dl 1775764853 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:3.0:1775764837.554318:0:21290:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f994f2e4380 x1862022967035648/t8589934593(8589934593) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 760/608 e 0 to 0 dl 1775764853 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1775764837.554578:0:21290:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1775764837.554601:0:21290:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0xed8:0x0], updating i_size 7680 00000002:00100000:3.0:1775764837.554700:0:21290:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f994f2e4380 x1862022967035648/t8589934593(8589934593) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 760/608 e 0 to 0 dl 1775764853 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:3.0:1775764837.555078:0:21290:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000401:0xed8:0x0](ffff8f9952eb4b08) cmd=c0086915 arg=20ad3d0 00000100:00100000:3.0:1775764837.555144:0:21290:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21290:1862022967035776:192.168.202.146@tcp:34:lfs.0 00000100:00100000:3.0:1775764837.555251:0:21290:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102f00 going to sleep for 11 seconds 00000100:00100000:0.0:1775764837.562980:0:21290:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21290:1862022967035776:192.168.202.146@tcp:34:lfs.0 00000080:00200008:0.0:1775764837.563197:0:21290:0:(file.c:447:ll_file_release()) START file d9.sanity-pfl:[0x200000401:0xed8:0x0](ffff8f9952eb4b08), flags 304000 00000002:00100000:0.0:1775764837.563258:0:21290:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f994f2e4380 x1862022967035648/t8589934593(8589934593) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 760/608 e 0 to 0 dl 1775764853 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1775764837.563269:0:21290:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1775764837.563273:0:21290:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970a75500 x1862022967036032/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/736 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1775764837.563297:0:21290:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21290:1862022967036032:192.168.202.146@tcp:35:lfs.0 00000100:00100000:0.0:1775764837.564351:0:21290:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9941370540 going to sleep for 11 seconds 00000100:00100000:0.0:1775764837.568604:0:21290:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21290:1862022967036032:192.168.202.146@tcp:35:lfs.0 00000002:00100000:0.0:1775764837.568641:0:21290:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f994f2e4380 x1862022967035648/t8589934593(8589934593) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 760/608 e 0 to 0 dl 1775764853 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00010000:00010000:0.0:1775764837.962550:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764838.985602:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764840.006047:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764841.051104:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764842.054892:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764842.181917:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967036288:192.168.202.146@tcp:400:kworker.0 00000100:00100000:1.0:1775764842.182026:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967036544:192.168.202.146@tcp:400:kworker.0 00000100:00100000:1.0:1775764842.182065:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1775764842.182072:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:884ae1b0-72f8-4a88-a129-db99cded7efc:2379:1862022967036160:192.168.202.146@tcp:400:kworker.0 00000100:00100000:1.0:1775764842.182097:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967036416:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764842.194624:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967036288:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764842.194644:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967036544:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764842.194717:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:884ae1b0-72f8-4a88-a129-db99cded7efc:2379:1862022967036160:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764842.194967:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967036416:192.168.202.146@tcp:400:kworker.0 00010000:00010000:0.0:1775764843.078467:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764844.101967:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:2.0:1775764844.845176:0:21858:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-pfl test 10: Inherit composite template from root ========================================================== 16:00:43 (1775764843) 00010000:00010000:3.0:1775764845.134050:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:1.0:1775764845.350956:0:21872:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8f9952d121c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764845.350971:0:21872:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764845.350995:0:21872:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764845.351015:0:21872:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764845.351054:0:21872:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1775764845.351063:0:21872:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764845.351072:0:21872:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764845.351076:0:21872:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c895 00000080:00010000:1.0:1775764845.351085:0:21872:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c895 from it ffffafbac1473cb8 00010000:00010000:1.0:1775764845.351089:0:21872:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764845.351098:0:21872:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:1.0:1775764845.351133:0:21872:0:(xattr.c:727:ll_xattr_get()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), xattr lov 00000100:00100000:1.0:1775764845.351186:0:21872:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job getfattr:3f83546c-01a9-4156-8a97-44c446245d3e:21872:1862022967036672:192.168.202.146@tcp:33:getfattr.0 00000100:00100000:1.0:1775764845.351418:0:21872:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413d80 going to sleep for 11 seconds 00000100:00100000:3.0:1775764845.358688:0:21872:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8f99523ee800: committing for last_committed 8589934594 gen 1 00000100:00100000:3.0:1775764845.358705:0:21872:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job getfattr:3f83546c-01a9-4156-8a97-44c446245d3e:21872:1862022967036672:192.168.202.146@tcp:33:getfattr.0 00000080:00200000:0.0:1775764845.415346:0:21876:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:0.0:1775764845.415375:0:21876:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764845.415394:0:21876:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:0.0:1775764845.415410:0:21876:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:0.0:1775764845.415418:0:21876:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764845.415425:0:21876:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:0.0:1775764845.415429:0:21876:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c895 00000080:00010000:0.0:1775764845.415437:0:21876:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c895 from it ffffafbac15ebb58 00010000:00010000:0.0:1775764845.415441:0:21876:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:0.0:1775764845.415450:0:21876:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:0.0:1775764845.415465:0:21876:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:0.0:1775764845.415478:0:21876:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=d10.sanity-pfl, dir=[0x200000007:0x1:0x0](ffff8f9952d121c8), flags=8 00000080:00200000:0.0:1775764845.415482:0:21876:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=d10.sanity-pfl, dir=[0x200000007:0x1:0x0](ffff8f9952d121c8), intent=getattr 00000002:00010000:0.0:1775764845.415495:0:21876:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: d10.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764845.415541:0:21876:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd800/0x36f9bdc9e421c8b1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21876 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764845.415583:0:21876:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd800/0x36f9bdc9e421c8b1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21876 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764845.415593:0:21876:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd800/0x36f9bdc9e421c8b1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21876 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764845.415620:0:21876:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:21876:1862022967036800:192.168.202.146@tcp:101:rm.0 00000100:00100000:0.0:1775764845.415713:0:21876:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f996687dd80 going to sleep for 11 seconds 00000100:00100000:0.0:1775764845.420373:0:21876:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:21876:1862022967036800:192.168.202.146@tcp:101:rm.0 00010000:00010000:0.0:1775764845.420396:0:21876:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd800/0x36f9bdc9e421c8b1 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21876 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764845.420409:0:21876:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd800/0x36f9bdc9e421c8b1 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21876 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764845.420415:0:21876:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd800/0x36f9bdc9e421c8b1 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21876 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764845.420421:0:21876:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd800/0x36f9bdc9e421c8b1 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21876 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764845.420429:0:21876:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd800/0x36f9bdc9e421c8b1 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21876 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764845.420435:0:21876:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99821cd800 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:0.0:1775764845.420440:0:21876:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd800/0x36f9bdc9e421c8b1 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 21876 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764845.420447:0:21876:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd800/0x36f9bdc9e421c8b1 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 21876 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764845.420478:0:21876:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd800/0x36f9bdc9e421c8b1 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21876 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764845.420485:0:21876:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1775764845.420491:0:21876:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd800/0x36f9bdc9e421c8b1 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21876 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764845.420498:0:21876:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd800/0x36f9bdc9e421c8b1 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21876 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764845.420514:0:21876:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=7, status=0 req@ffff8f9970a75500 x1862022967036800/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/584 e 0 to 0 dl 1775764861 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1775764845.420528:0:21876:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:0.0:1775764845.420555:0:21876:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:0.0:1775764845.420568:0:21876:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00010000:0.0:1775764845.420576:0:21876:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c895 from it ffffafbac15eb930 00010000:00010000:0.0:1775764845.420580:0:21876:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:0.0:1775764845.420588:0:21876:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:0.0:1775764845.420677:0:21876:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:0.0:1775764845.420690:0:21876:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764845.420712:0:21876:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:0.0:1775764845.420720:0:21876:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:0.0:1775764845.420727:0:21876:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764845.420732:0:21876:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:0.0:1775764845.420735:0:21876:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c895 00000080:00010000:0.0:1775764845.420769:0:21876:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c895 from it ffffafbac15ebc10 00010000:00010000:0.0:1775764845.420772:0:21876:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:0.0:1775764845.420779:0:21876:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:0.0:1775764845.420788:0:21876:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:0.0:1775764845.420798:0:21876:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d10.sanity-pfl, flags=0 00000080:00200000:0.0:1775764845.428968:0:21877:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:0.0:1775764845.428997:0:21877:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764845.429015:0:21877:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:0.0:1775764845.429031:0:21877:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:0.0:1775764845.429039:0:21877:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764845.429048:0:21877:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:0.0:1775764845.429052:0:21877:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c895 00000080:00010000:0.0:1775764845.429095:0:21877:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c895 from it ffffafbac16d3bb0 00010000:00010000:0.0:1775764845.429099:0:21877:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:0.0:1775764845.429109:0:21877:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:0.0:1775764845.429124:0:21877:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 44 00000080:00200008:0.0:1775764845.429132:0:21877:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8f9952d121c8), flags 320400 00000080:00200080:0.0:1775764845.429666:0:21877:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=4008669a arg=127d2e0 00010000:00010000:0.0:1775764845.429688:0:21877:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:0.0:1775764845.429920:0:21877:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:0.0:1775764845.429936:0:21877:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764845.429939:0:21877:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5e00/0x36f9bdc9e421c895 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fbf17 expref: -99 pid: 20829 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000020:00100000:0.0:1775764845.429961:0:21877:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1775764845.429974:0:21877:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970a75500 x1862022967037056/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764845.430002:0:21877:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21877:1862022967037056:192.168.202.146@tcp:36:lfs.0 00000100:00100000:0.0:1775764845.430186:0:21877:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f996687dd80 going to sleep for 11 seconds 00000100:00100000:3.0:1775764845.455022:0:21877:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21877:1862022967037056:192.168.202.146@tcp:36:lfs.0 00000080:00200008:3.0:1775764845.455085:0:21877:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8f9952d121c8), flags 320000 00000080:00200008:2.0:1775764845.483453:0:21878:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8f9952d121c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764845.483467:0:21878:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:2.0:1775764845.483487:0:21878:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764845.483550:0:21878:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8b8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21878 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.483560:0:21878:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8b8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21878 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.483569:0:21878:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8b8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21878 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764845.483595:0:21878:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21878:1862022967037184:192.168.202.146@tcp:101:lfs.0 00000100:00100000:2.0:1775764845.483706:0:21878:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9951fa93c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764845.490707:0:21878:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21878:1862022967037184:192.168.202.146@tcp:101:lfs.0 00010000:00010000:2.0:1775764845.490728:0:21878:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8b8 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfa3 expref: -99 pid: 21878 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.490770:0:21878:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8b8 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfa3 expref: -99 pid: 21878 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.490783:0:21878:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8b8 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbfa3 expref: -99 pid: 21878 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.490795:0:21878:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8b8 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfa3 expref: -99 pid: 21878 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.490808:0:21878:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8b8 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbfa3 expref: -99 pid: 21878 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.490815:0:21878:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8b8 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbfa3 expref: -99 pid: 21878 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.490823:0:21878:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8b8 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbfa3 expref: -99 pid: 21878 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764845.490833:0:21878:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f99703d3b80 x1862022967037184/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764861 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764845.490847:0:21878:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8b8 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfa3 expref: -99 pid: 21878 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.490860:0:21878:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1775764845.491007:0:21878:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764845.491034:0:21878:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 8704 00000080:00010000:2.0:1775764845.491041:0:21878:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c8b8 00000080:00010000:2.0:1775764845.491047:0:21878:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:2.0:1775764845.491050:0:21878:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c8b8 from it ffffafbac1d73cb8 00010000:00010000:2.0:1775764845.491053:0:21878:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8b8 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbfa3 expref: -99 pid: 21878 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.491061:0:21878:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8b8 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfa3 expref: -99 pid: 21878 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764845.491136:0:21878:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:2.0:1775764845.491236:0:21878:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764845.491245:0:21878:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8b8 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfa3 expref: -99 pid: 21878 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764845.491255:0:21878:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8b8 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfa3 expref: -99 pid: 21878 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764845.491261:0:21878:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764845.491266:0:21878:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:2.0:1775764845.491268:0:21878:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c8b8 00000080:00010000:2.0:1775764845.491271:0:21878:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c8b8 from it ffffafbac1d73bb0 00010000:00010000:2.0:1775764845.491274:0:21878:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8b8 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbfa3 expref: -99 pid: 21878 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764845.491281:0:21878:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8b8 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfa3 expref: -99 pid: 21878 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764845.491291:0:21878:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 44 00000080:00200008:2.0:1775764845.491300:0:21878:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8f9952d121c8), flags 300000 00000080:00200000:2.0:1775764845.491362:0:21878:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:2.0:1775764845.491367:0:21878:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764845.491372:0:21878:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8b8 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfa3 expref: -99 pid: 21878 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764845.491377:0:21878:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8b8 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfa3 expref: -99 pid: 21878 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764845.491381:0:21878:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764845.491384:0:21878:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:2.0:1775764845.491385:0:21878:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c8b8 00000080:00010000:2.0:1775764845.491388:0:21878:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c8b8 from it ffffafbac1d73c18 00010000:00010000:2.0:1775764845.491389:0:21878:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8b8 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbfa3 expref: -99 pid: 21878 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764845.491393:0:21878:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8b8 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfa3 expref: -99 pid: 21878 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764845.491398:0:21878:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 2 00000080:00200000:2.0:1775764845.491405:0:21878:0:(xattr.c:410:ll_xattr_set()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), xattr lov 00010000:00010000:2.0:1775764845.491415:0:21878:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8b8 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fbfa3 expref: -99 pid: 21878 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764845.491450:0:21878:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8b8 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fbfa3 expref: -99 pid: 21878 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764845.491457:0:21878:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764845.491459:0:21878:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8b8 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fbfa3 expref: -99 pid: 21878 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:2.0:1775764845.491481:0:21878:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1775764845.491489:0:21878:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f99703d3b80 x1862022967037440/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 760/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764845.491511:0:21878:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21878:1862022967037440:192.168.202.146@tcp:36:lfs.0 00000100:00100000:2.0:1775764845.491651:0:21878:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9951fa93c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764845.499660:0:21878:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21878:1862022967037440:192.168.202.146@tcp:36:lfs.0 00000080:00200008:2.0:1775764845.499754:0:21878:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8f9952d121c8), flags 300000 00000080:00200000:0.0:1775764845.541999:0:21883:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:0.0:1775764845.542022:0:21883:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764845.542085:0:21883:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c8bf lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21883 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764845.542144:0:21883:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c8bf lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21883 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764845.542155:0:21883:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c8bf lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21883 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764845.542181:0:21883:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job mkdir:3f83546c-01a9-4156-8a97-44c446245d3e:21883:1862022967037568:192.168.202.146@tcp:101:mkdir.0 00000100:00100000:0.0:1775764845.542284:0:21883:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f996687dcc0 going to sleep for 11 seconds 00000100:00100000:0.0:1775764845.570516:0:21883:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job mkdir:3f83546c-01a9-4156-8a97-44c446245d3e:21883:1862022967037568:192.168.202.146@tcp:101:mkdir.0 00010000:00010000:0.0:1775764845.570544:0:21883:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c8bf lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfb8 expref: -99 pid: 21883 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764845.570558:0:21883:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c8bf lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfb8 expref: -99 pid: 21883 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764845.570573:0:21883:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c8bf lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbfb8 expref: -99 pid: 21883 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764845.570584:0:21883:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c8bf lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfb8 expref: -99 pid: 21883 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764845.570596:0:21883:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c8bf lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbfb8 expref: -99 pid: 21883 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764845.570604:0:21883:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c8bf lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbfb8 expref: -99 pid: 21883 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764845.570655:0:21883:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c8bf lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbfb8 expref: -99 pid: 21883 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764845.570667:0:21883:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970a77800 x1862022967037568/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764861 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'mkdir.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1775764845.570683:0:21883:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c8bf lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfb8 expref: -99 pid: 21883 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764845.570698:0:21883:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1775764845.570707:0:21883:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1775764845.570781:0:21883:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 8704 00000080:00010000:0.0:1775764845.570789:0:21883:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c8bf 00000080:00010000:0.0:1775764845.570795:0:21883:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:0.0:1775764845.570798:0:21883:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c8bf from it ffffafbac1d6bbe0 00010000:00010000:0.0:1775764845.570801:0:21883:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c8bf lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbfb8 expref: -99 pid: 21883 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764845.570809:0:21883:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c8bf lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfb8 expref: -99 pid: 21883 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764845.570837:0:21883:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:0.0:1775764845.570864:0:21883:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=d10.sanity-pfl, dir=[0x200000007:0x1:0x0](ffff8f9952d121c8), flags=1536 00000080:00200000:0.0:1775764845.570869:0:21883:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:0.0:1775764845.570885:0:21883:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764845.570894:0:21883:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c8bf lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfb8 expref: -99 pid: 21883 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764845.570924:0:21883:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c8bf lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfb8 expref: -99 pid: 21883 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764845.570941:0:21883:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764845.570946:0:21883:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:0.0:1775764845.570949:0:21883:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c8bf 00000080:00010000:0.0:1775764845.570952:0:21883:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c8bf from it ffffafbac1d6bce0 00010000:00010000:0.0:1775764845.570955:0:21883:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c8bf lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbfb8 expref: -99 pid: 21883 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764845.570962:0:21883:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c8bf lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfb8 expref: -99 pid: 21883 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764845.570971:0:21883:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 3 00000080:00200000:0.0:1775764845.570977:0:21883:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:0.0:1775764845.570983:0:21883:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764845.570989:0:21883:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c8bf lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfb8 expref: -99 pid: 21883 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764845.570997:0:21883:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c8bf lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfb8 expref: -99 pid: 21883 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764845.571003:0:21883:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764845.571007:0:21883:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:0.0:1775764845.571009:0:21883:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c8bf 00000080:00010000:0.0:1775764845.571012:0:21883:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c8bf from it ffffafbac1d6bdc8 00010000:00010000:0.0:1775764845.571029:0:21883:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c8bf lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbfb8 expref: -99 pid: 21883 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764845.571036:0:21883:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c8bf lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfb8 expref: -99 pid: 21883 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1775764845.571044:0:21883:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 3 00000080:00200000:0.0:1775764845.571051:0:21883:0:(namei.c:2216:do_mkdir()) VFS Op:name=d10.sanity-pfl, dir=[0x200000007:0x1:0x0](ffff8f9952d121c8) 00010000:00010000:0.0:1775764845.571089:0:21883:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c8bf lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fbfb8 expref: -99 pid: 21883 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764845.571134:0:21883:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c8bf lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fbfb8 expref: -99 pid: 21883 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764845.571141:0:21883:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764845.571144:0:21883:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc800/0x36f9bdc9e421c8bf lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fbfb8 expref: -99 pid: 21883 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:0.0:1775764845.571169:0:21883:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1775764845.571176:0:21883:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970a77800 x1862022967037824/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 560/688 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764845.571201:0:21883:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job mkdir:3f83546c-01a9-4156-8a97-44c446245d3e:21883:1862022967037824:192.168.202.146@tcp:36:mkdir.0 00000100:00100000:0.0:1775764845.571295:0:21883:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f996687dcc0 going to sleep for 11 seconds 00000100:00100000:0.0:1775764845.613451:0:21883:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job mkdir:3f83546c-01a9-4156-8a97-44c446245d3e:21883:1862022967037824:192.168.202.146@tcp:36:mkdir.0 00000080:00200000:0.0:1775764845.613510:0:21883:0:(llite_lib.c:3177:ll_read_inode2()) VFS Op:inode=[0x200000401:0xedc:0x0](ffff8f9952e49148) 00000080:00200000:0.0:1775764845.613517:0:21883:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0xedc:0x0], updating i_size 512 00000080:00200000:3.0:1775764845.627409:0:21884:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:3.0:1775764845.627440:0:21884:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764845.627595:0:21884:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21884 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764845.627983:0:21884:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21884 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764845.627998:0:21884:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21884 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764845.628025:0:21884:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21884:1862022967037952:192.168.202.146@tcp:101:lfs.0 00000100:00100000:3.0:1775764845.628127:0:21884:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560d80 going to sleep for 11 seconds 00000100:00100000:3.0:1775764845.640879:0:21884:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21884:1862022967037952:192.168.202.146@tcp:101:lfs.0 00010000:00010000:3.0:1775764845.640906:0:21884:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764845.640919:0:21884:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764845.640940:0:21884:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764845.640953:0:21884:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764845.640973:0:21884:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764845.640981:0:21884:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764845.641013:0:21884:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764845.641025:0:21884:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970edc700 x1862022967037952/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764861 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1775764845.641039:0:21884:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764845.641052:0:21884:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1775764845.641058:0:21884:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1775764845.641089:0:21884:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 8704 00000080:00010000:3.0:1775764845.641095:0:21884:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c8c6 00000080:00010000:3.0:1775764845.641100:0:21884:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:3.0:1775764845.641103:0:21884:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c8c6 from it ffffafbac17abb58 00010000:00010000:3.0:1775764845.641106:0:21884:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764845.641113:0:21884:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764845.641135:0:21884:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:3.0:1775764845.641155:0:21884:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=d10.sanity-pfl, dir=[0x200000007:0x1:0x0](ffff8f9952d121c8), flags=8 00000080:00200000:3.0:1775764845.641159:0:21884:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=d10.sanity-pfl, dir=[0x200000007:0x1:0x0](ffff8f9952d121c8), intent=getattr 00000002:00010000:3.0:1775764845.641177:0:21884:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: d10.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764845.641214:0:21884:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4400/0x36f9bdc9e421c8cd lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21884 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764845.641239:0:21884:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4400/0x36f9bdc9e421c8cd lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21884 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764845.641259:0:21884:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4400/0x36f9bdc9e421c8cd lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21884 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764845.641281:0:21884:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21884:1862022967038080:192.168.202.146@tcp:101:lfs.0 00000100:00100000:3.0:1775764845.641374:0:21884:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560d80 going to sleep for 11 seconds 00000100:00100000:3.0:1775764845.646334:0:21884:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21884:1862022967038080:192.168.202.146@tcp:101:lfs.0 00010000:00010000:3.0:1775764845.646363:0:21884:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4400/0x36f9bdc9e421c8cd lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbff0 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764845.646388:0:21884:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4400/0x36f9bdc9e421c8cd lrc: 4/1,0 mode: --/PR res: [0x200000401:0xedc:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbff0 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764845.646397:0:21884:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4400/0x36f9bdc9e421c8cd lrc: 4/1,0 mode: --/PR res: [0x200000401:0xedc:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbff0 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764845.646408:0:21884:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4400/0x36f9bdc9e421c8cd lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xedc:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbff0 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764845.646418:0:21884:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4400/0x36f9bdc9e421c8cd lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xedc:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbff0 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764845.646430:0:21884:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4400/0x36f9bdc9e421c8cd lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xedc:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbff0 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764845.646491:0:21884:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4400/0x36f9bdc9e421c8cd lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xedc:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbff0 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764845.646498:0:21884:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4400/0x36f9bdc9e421c8cd lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xedc:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbff0 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764845.646513:0:21884:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970edc700 x1862022967038080/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/584 e 0 to 0 dl 1775764861 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1775764845.646524:0:21884:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4400/0x36f9bdc9e421c8cd lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xedc:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbff0 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764845.646534:0:21884:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/3804 (0 0) 00000080:00200000:3.0:1775764845.646541:0:21884:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:3.0:1775764845.646563:0:21884:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0xedc:0x0], updating i_size 512 00000080:00200000:3.0:1775764845.646567:0:21884:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0xedc:0x0](ffff8f9952e49148): rc = 0 00000080:00010000:3.0:1775764845.646572:0:21884:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xedc:0x0](ffff8f9952e49148) for lock 0x36f9bdc9e421c8cd 00000080:00010000:3.0:1775764845.646586:0:21884:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xedc:0x0](ffff8f9952e49148) 00000080:00010000:3.0:1775764845.646590:0:21884:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c8cd from it ffffafbac17abb20 00010000:00010000:3.0:1775764845.646593:0:21884:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4400/0x36f9bdc9e421c8cd lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xedc:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbff0 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764845.646601:0:21884:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4400/0x36f9bdc9e421c8cd lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xedc:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbff0 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:3.0:1775764845.647003:0:21884:0:(file.c:6226:ll_getattr_dentry()) START file d10.sanity-pfl:[0x200000401:0xedc:0x0](ffff8f9952e49148), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764845.647115:0:21884:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xedc:0x0](ffff8f9952e49148),name=d10.sanity-pfl 00000002:00010000:2.0:1775764845.647981:0:21884:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xedc:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764845.647999:0:21884:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4400/0x36f9bdc9e421c8cd lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xedc:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbff0 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764845.648014:0:21884:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4400/0x36f9bdc9e421c8cd lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xedc:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbff0 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764845.648022:0:21884:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764845.648029:0:21884:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xedc:0x0](ffff8f9952e49148) 00000080:00010000:2.0:1775764845.648032:0:21884:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xedc:0x0](ffff8f9952e49148) for lock 0x36f9bdc9e421c8cd 00000080:00010000:2.0:1775764845.648037:0:21884:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c8cd from it ffffafbac17abcb8 00010000:00010000:2.0:1775764845.648041:0:21884:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4400/0x36f9bdc9e421c8cd lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xedc:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbff0 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764845.648049:0:21884:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4400/0x36f9bdc9e421c8cd lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xedc:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbff0 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764845.648108:0:21884:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:2.0:1775764845.648116:0:21884:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764845.648123:0:21884:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764845.648131:0:21884:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764845.648144:0:21884:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764845.648148:0:21884:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:2.0:1775764845.648151:0:21884:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c8c6 00000080:00010000:2.0:1775764845.648154:0:21884:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c8c6 from it ffffafbac17abbf8 00010000:00010000:2.0:1775764845.648157:0:21884:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764845.648163:0:21884:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764845.648171:0:21884:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:2.0:1775764845.648177:0:21884:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d10.sanity-pfl, flags=267 00000080:00200000:2.0:1775764845.648183:0:21884:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedc:0x0](ffff8f9952e49148), inode mode 41ed mask 44 00000080:00200008:2.0:1775764845.648191:0:21884:0:(file.c:1057:ll_file_open()) START file d10.sanity-pfl:[0x200000401:0xedc:0x0](ffff8f9952e49148), flags 300000 00000002:00010000:2.0:1775764845.648206:0:21884:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xedc:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0240300001 00010000:00010000:2.0:1775764845.648252:0:21884:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8d4 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21884 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.648260:0:21884:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8d4 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21884 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764845.648271:0:21884:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764845.648281:0:21884:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f99703d0a80 x1862022967038336/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 760/131968 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764845.648291:0:21884:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8d4 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21884 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764845.648312:0:21884:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0a80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21884:1862022967038336:192.168.202.146@tcp:101:lfs.0 00000100:00100000:2.0:1775764845.648518:0:21884:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9951fa9a80 going to sleep for 11 seconds 00000100:00100000:2.0:1775764845.657731:0:21884:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0a80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21884:1862022967038336:192.168.202.146@tcp:101:lfs.0 00010000:00010000:2.0:1775764845.657797:0:21884:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8d4 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21884 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.657811:0:21884:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8d4 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21884 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.657818:0:21884:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8d4 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21884 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.657825:0:21884:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8d4 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21884 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.657833:0:21884:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8d4 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21884 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.657839:0:21884:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99706ea800 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:2.0:1775764845.657843:0:21884:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8d4 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21884 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.657850:0:21884:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8d4 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21884 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.657857:0:21884:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8d4 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21884 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.657873:0:21884:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1775764845.657879:0:21884:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8d4 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21884 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.657889:0:21884:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8d4 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21884 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764845.657908:0:21884:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8f99703d0a80 x1862022967038336/t8589934598(8589934598) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 760/608 e 0 to 0 dl 1775764861 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764845.657920:0:21884:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f99703d0a80 x1862022967038336/t8589934598(8589934598) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 760/608 e 0 to 0 dl 1775764861 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764845.657942:0:21884:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764845.657964:0:21884:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0xedc:0x0], updating i_size 512 00000002:00100000:2.0:1775764845.657987:0:21884:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f99703d0a80 x1862022967038336/t8589934598(8589934598) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 760/608 e 0 to 0 dl 1775764861 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764845.658092:0:21884:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedc:0x0](ffff8f9952e49148), inode mode 41ed mask 2 00000080:00200000:2.0:1775764845.658103:0:21884:0:(xattr.c:410:ll_xattr_set()) VFS Op:inode=[0x200000401:0xedc:0x0](ffff8f9952e49148), xattr lov 00010000:00010000:2.0:1775764845.658126:0:21884:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4400/0x36f9bdc9e421c8cd lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xedc:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fbff0 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764845.658178:0:21884:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4400/0x36f9bdc9e421c8cd lrc: 1/0,0 mode: --/PR res: [0x200000401:0xedc:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fbff0 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764845.658185:0:21884:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764845.658188:0:21884:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4400/0x36f9bdc9e421c8cd lrc: 0/0,0 mode: --/PR res: [0x200000401:0xedc:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fbff0 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000020:00100000:2.0:1775764845.658207:0:21884:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1775764845.658213:0:21884:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f99703d1880 x1862022967038592/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 600/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764845.658238:0:21884:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21884:1862022967038592:192.168.202.146@tcp:36:lfs.0 00000100:00100000:2.0:1775764845.658338:0:21884:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9951fa9a80 going to sleep for 11 seconds 00000100:00100000:2.0:1775764845.668545:0:21884:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21884:1862022967038592:192.168.202.146@tcp:36:lfs.0 00000080:00200008:2.0:1775764845.668615:0:21884:0:(file.c:447:ll_file_release()) START file d10.sanity-pfl:[0x200000401:0xedc:0x0](ffff8f9952e49148), flags 300000 00000002:00100000:2.0:1775764845.668673:0:21884:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f99703d0a80 x1862022967038336/t8589934598(8589934598) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 760/608 e 0 to 0 dl 1775764861 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764845.668683:0:21884:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764845.668687:0:21884:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f99703d3100 x1862022967038848/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/736 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764845.668707:0:21884:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21884:1862022967038848:192.168.202.146@tcp:35:lfs.0 00000100:00100000:2.0:1775764845.668814:0:21884:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9951fa9a80 going to sleep for 11 seconds 00000100:00100000:2.0:1775764845.680276:0:21884:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21884:1862022967038848:192.168.202.146@tcp:35:lfs.0 00000002:00100000:2.0:1775764845.680312:0:21884:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f99703d0a80 x1862022967038336/t8589934598(8589934598) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 760/608 e 0 to 0 dl 1775764861 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764845.690099:0:21885:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:2.0:1775764845.690132:0:21885:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764845.690155:0:21885:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764845.690175:0:21885:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764845.690184:0:21885:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764845.690194:0:21885:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:2.0:1775764845.690198:0:21885:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c8c6 00000080:00010000:2.0:1775764845.690206:0:21885:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c8c6 from it ffffafbac1827bf8 00010000:00010000:2.0:1775764845.690210:0:21885:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764845.690219:0:21885:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764845.690238:0:21885:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:2.0:1775764845.690251:0:21885:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=d10.sanity-pfl, dir=[0x200000007:0x1:0x0](ffff8f9952d121c8), flags=25 00000080:00200000:2.0:1775764845.690257:0:21885:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=d10.sanity-pfl, dir=[0x200000007:0x1:0x0](ffff8f9952d121c8), intent=0 00000002:00010000:2.0:1775764845.690269:0:21885:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: d10.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: lookup flags 0 00010000:00010000:2.0:1775764845.690317:0:21885:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8db lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21885 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.690326:0:21885:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8db lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21885 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.690336:0:21885:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8db lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21885 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764845.690365:0:21885:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job touch:3f83546c-01a9-4156-8a97-44c446245d3e:21885:1862022967038976:192.168.202.146@tcp:101:touch.0 00000100:00100000:2.0:1775764845.690474:0:21885:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9951fa9a80 going to sleep for 11 seconds 00000100:00100000:2.0:1775764845.711510:0:21885:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job touch:3f83546c-01a9-4156-8a97-44c446245d3e:21885:1862022967038976:192.168.202.146@tcp:101:touch.0 00010000:00010000:2.0:1775764845.711536:0:21885:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8db lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc021 expref: -99 pid: 21885 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.711565:0:21885:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8db lrc: 4/1,0 mode: --/PR res: [0x200000401:0xedc:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc021 expref: -99 pid: 21885 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.711574:0:21885:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8db lrc: 4/1,0 mode: --/PR res: [0x200000401:0xedc:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc021 expref: -99 pid: 21885 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.711586:0:21885:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8db lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xedc:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc021 expref: -99 pid: 21885 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.711597:0:21885:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8db lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xedc:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc021 expref: -99 pid: 21885 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.711609:0:21885:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8db lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xedc:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc021 expref: -99 pid: 21885 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.711617:0:21885:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8db lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xedc:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc021 expref: -99 pid: 21885 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.711625:0:21885:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8db lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xedc:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc021 expref: -99 pid: 21885 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764845.711636:0:21885:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=b, status=0 req@ffff8f99703d3100 x1862022967038976/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/584 e 0 to 0 dl 1775764861 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764845.711648:0:21885:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8db lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xedc:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc021 expref: -99 pid: 21885 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.711657:0:21885:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/3804 (0 0) 00000080:00200000:2.0:1775764845.711663:0:21885:0:(namei.c:943:ll_intent_lock()) intent lock 16 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764845.711684:0:21885:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0xedc:0x0], updating i_size 512 00000080:00200000:2.0:1775764845.711687:0:21885:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0xedc:0x0](ffff8f9952e49148): rc = 0 00000080:00010000:2.0:1775764845.711691:0:21885:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xedc:0x0](ffff8f9952e49148) for lock 0x36f9bdc9e421c8db 00000080:00010000:2.0:1775764845.711703:0:21885:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xedc:0x0](ffff8f9952e49148) 00000080:00010000:2.0:1775764845.711706:0:21885:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c8db from it ffffafbac1827ac8 00010000:00010000:2.0:1775764845.711709:0:21885:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8db lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xedc:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc021 expref: -99 pid: 21885 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.711717:0:21885:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea800/0x36f9bdc9e421c8db lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xedc:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc021 expref: -99 pid: 21885 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764845.711787:0:21885:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedc:0x0](ffff8f9952e49148), inode mode 41ed mask 1 00000080:00200000:2.0:1775764845.711805:0:21885:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedc:0x0](ffff8f9952e49148), inode mode 41ed mask 3 00000080:00200000:2.0:1775764845.711811:0:21885:0:(namei.c:1399:ll_atomic_open()) VFS Op:name=file1, dir=[0x200000401:0xedc:0x0](ffff8f9952e49148), file ffff8f99429c6600, open_flags 8941, mode 81b6 opened 0 00000080:00200000:2.0:1775764845.711818:0:21885:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=file1, dir=[0x200000401:0xedc:0x0](ffff8f9952e49148), intent=open|creat 00000002:00010000:2.0:1775764845.711841:0:21885:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: file1,[0x200000401:0xedd:0x0]) in obj [0x200000401:0xedc:0x0], intent: open|creat flags 0104502 00010000:00010000:2.0:1775764845.711899:0:21885:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c8e2 lrc: 3/0,1 mode: --/CW res: [0x200000401:0xedc:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21885 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.711906:0:21885:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c8e2 lrc: 3/0,1 mode: --/CW res: [0x200000401:0xedc:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21885 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764845.711916:0:21885:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764845.711922:0:21885:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f99703d3100 x1862022967039232/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 760/66432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764845.711943:0:21885:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c8e2 lrc: 3/0,1 mode: --/CW res: [0x200000401:0xedc:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21885 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764845.711968:0:21885:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job touch:3f83546c-01a9-4156-8a97-44c446245d3e:21885:1862022967039232:192.168.202.146@tcp:101:touch.0 00000100:00100000:2.0:1775764845.712327:0:21885:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9951fa9a80 going to sleep for 11 seconds 00000100:00100000:2.0:1775764845.729119:0:21885:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job touch:3f83546c-01a9-4156-8a97-44c446245d3e:21885:1862022967039232:192.168.202.146@tcp:101:touch.0 00010000:00010000:2.0:1775764845.729149:0:21885:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c8e2 lrc: 4/0,1 mode: --/CW res: [0x200000401:0xedc:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc03d expref: -99 pid: 21885 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.729175:0:21885:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c8e2 lrc: 4/0,1 mode: --/PR res: [0x200000401:0xedd:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc03d expref: -99 pid: 21885 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.729184:0:21885:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c8e2 lrc: 4/0,1 mode: --/PR res: [0x200000401:0xedd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc03d expref: -99 pid: 21885 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.729196:0:21885:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c8e2 lrc: 4/0,1 mode: PR/PR res: [0x200000401:0xedd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc03d expref: -99 pid: 21885 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.729209:0:21885:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c8e2 lrc: 4/0,1 mode: PR/PR res: [0x200000401:0xedd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc03d expref: -99 pid: 21885 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.729223:0:21885:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c8e2 lrc: 5/1,1 mode: PR/PR res: [0x200000401:0xedd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc03d expref: -99 pid: 21885 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.729232:0:21885:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c8e2 lrc: 5/1,1 mode: PR/PR res: [0x200000401:0xedd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc03d expref: -99 pid: 21885 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.729239:0:21885:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c8e2 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xedd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc03d expref: -99 pid: 21885 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764845.729256:0:21885:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=3 disposition=37, status=0 req@ffff8f99703d3100 x1862022967039232/t8589934601(8589934601) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 760/744 e 0 to 0 dl 1775764861 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764845.729269:0:21885:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f99703d3100 x1862022967039232/t8589934601(8589934601) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 760/744 e 0 to 0 dl 1775764861 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764845.729283:0:21885:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 136 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c8e2 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xedd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc03d expref: -99 pid: 21885 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764845.729295:0:21885:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c8e2 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xedd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc03d expref: -99 pid: 21885 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.729306:0:21885:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/3805 (0 0) 00000080:00200000:2.0:1775764845.729313:0:21885:0:(namei.c:943:ll_intent_lock()) intent lock 3 on i1 [0x200000401:0xedc:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764845.729326:0:21885:0:(namei.c:1610:ll_create_it()) VFS Op:name=file1, dir=[0x200000401:0xedc:0x0](ffff8f9952e49148), intent=open|creat 00000080:00200000:2.0:1775764845.729356:0:21885:0:(llite_lib.c:3177:ll_read_inode2()) VFS Op:inode=[0x200000401:0xedd:0x0](ffff8f9952eb1988) 00000080:00200000:2.0:1775764845.729529:0:21885:0:(file.c:6766:ll_layout_conf()) [0x200000401:0xedd:0x0]: layout version change: 4294967294 -> 1 00000080:00010000:2.0:1775764845.729539:0:21885:0:(namei.c:1576:ll_create_node()) setting l_ast_data to inode [0x200000401:0xedd:0x0](ffff8f9952eb1988) 00000080:00010000:2.0:1775764845.729542:0:21885:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xedd:0x0](ffff8f9952eb1988) for lock 0x36f9bdc9e421c8e2 00000080:00010000:2.0:1775764845.729551:0:21885:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xedd:0x0](ffff8f9952eb1988) 00000080:00200008:2.0:1775764845.729566:0:21885:0:(file.c:1057:ll_file_open()) START file file1:[0x200000401:0xedd:0x0](ffff8f9952eb1988), flags 104501 00000002:00100000:2.0:1775764845.729586:0:21885:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f99703d3100 x1862022967039232/t8589934601(8589934601) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 656/744 e 0 to 0 dl 1775764861 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1775764845.729632:0:21885:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c8e2 from it ffff8f9970a69de0 00010000:00010000:2.0:1775764845.729637:0:21885:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c8e2 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xedd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fc03d expref: -99 pid: 21885 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.729652:0:21885:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c8e2 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xedd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fc03d expref: -99 pid: 21885 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764845.729673:0:21885:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedd:0x0](ffff8f9952eb1988), inode mode 81a4 mask 40 00000080:00200008:2.0:1775764845.729712:0:21885:0:(llite_lib.c:2274:ll_setattr_raw()) START file file1:[0x200000401:0xedd:0x0](ffff8f9952eb1988) current size 0, valid attrs 20070, mode 8f99, uid -1048412592, gid -20550, new size 18446744071971469847, atime 1775764845.000000000, mtime 1775764845.000000000, ctime 1775764845.000000000 00010000:00010000:2.0:1775764845.729731:0:21885:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c8e2 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xedd:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5fc03d expref: -99 pid: 21885 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764845.729814:0:21885:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0xedd:0x0]: losing layout lock 00010000:00010000:2.0:1775764845.729858:0:21885:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c8e2 lrc: 1/0,0 mode: --/PR res: [0x200000401:0xedd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fc03d expref: -99 pid: 21885 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764845.729879:0:21885:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764845.729882:0:21885:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c8e2 lrc: 0/0,0 mode: --/PR res: [0x200000401:0xedd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fc03d expref: -99 pid: 21885 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000020:00100000:2.0:1775764845.729906:0:21885:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1775764845.729912:0:21885:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f99703d0a80 x1862022967039488/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764845.729946:0:21885:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0a80 pname:cluuid:pid:xid:nid:opc:job touch:3f83546c-01a9-4156-8a97-44c446245d3e:21885:1862022967039488:192.168.202.146@tcp:36:touch.0 00000100:00100000:2.0:1775764845.730036:0:21885:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9951fa9a80 going to sleep for 11 seconds 00000100:00100000:2.0:1775764845.737569:0:21885:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0a80 pname:cluuid:pid:xid:nid:opc:job touch:3f83546c-01a9-4156-8a97-44c446245d3e:21885:1862022967039488:192.168.202.146@tcp:36:touch.0 00000080:00200000:2.0:1775764845.737656:0:21885:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xedd:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00010000:2.0:1775764845.737677:0:21885:0:(file.c:6988:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200000401:0xedd:0x0](ffff8f9952eb1988) 00000002:00010000:2.0:1775764845.737687:0:21885:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xedd:0x0]) in obj [0x200000401:0xedd:0x0], intent: layout flags 0 00010000:00010000:2.0:1775764845.737736:0:21885:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c8e9 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedd:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21885 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.737763:0:21885:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c8e9 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedd:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21885 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.737773:0:21885:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c8e9 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedd:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21885 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764845.737793:0:21885:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job touch:3f83546c-01a9-4156-8a97-44c446245d3e:21885:1862022967039616:192.168.202.146@tcp:101:touch.0 00000100:00100000:2.0:1775764845.737881:0:21885:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9951fa9a80 going to sleep for 11 seconds 00000100:00100000:2.0:1775764845.756770:0:21885:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job touch:3f83546c-01a9-4156-8a97-44c446245d3e:21885:1862022967039616:192.168.202.146@tcp:101:touch.0 00010000:00010000:2.0:1775764845.756803:0:21885:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c8e9 lrc: 4/1,0 mode: CR/CR res: [0x200000401:0xedd:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc052 expref: -99 pid: 21885 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.756819:0:21885:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c8e9 lrc: 4/1,0 mode: CR/CR res: [0x200000401:0xedd:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc052 expref: -99 pid: 21885 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764845.756834:0:21885:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8f99703d3480 x1862022967039616/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 376/480 e 0 to 0 dl 1775764861 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764845.756845:0:21885:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 136 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c8e9 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xedd:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc052 expref: -99 pid: 21885 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764845.756857:0:21885:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c8e9 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xedd:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc052 expref: -99 pid: 21885 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.756879:0:21885:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 16 res 8589935617/3805 (0 0) 00000080:00200000:2.0:1775764845.756886:0:21885:0:(namei.c:943:ll_intent_lock()) intent lock 1024 on i1 [0x200000401:0xedd:0x0] suppgids 0 0: rc 0 00000080:00010000:2.0:1775764845.756904:0:21885:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xedd:0x0](ffff8f9952eb1988) for lock 0x36f9bdc9e421c8e9 00000080:00010000:2.0:1775764845.756912:0:21885:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c8e9 from it ffffafbac1827a78 00010000:00010000:2.0:1775764845.756917:0:21885:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c8e9 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xedd:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc052 expref: -99 pid: 21885 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764845.756935:0:21885:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c8e9 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xedd:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc052 expref: -99 pid: 21885 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764845.756959:0:21885:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c8e9 lrc: 2/1,0 mode: CR/CR res: [0x200000401:0xedd:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc052 expref: -99 pid: 21885 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764845.756970:0:21885:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c8e9 lrc: 2/1,0 mode: CR/CR res: [0x200000401:0xedd:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc052 expref: -99 pid: 21885 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:2.0:1775764845.756978:0:21885:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200000401:0xedd:0x0](00000000400837aa) being reconfigured ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c8e9 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xedd:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc052 expref: -99 pid: 21885 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764845.757047:0:21885:0:(file.c:6766:ll_layout_conf()) [0x200000401:0xedd:0x0]: layout version change: 4294967294 -> 1 00010000:00010000:2.0:1775764845.757069:0:21885:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c8e9 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xedd:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fc052 expref: -99 pid: 21885 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764845.757075:0:21885:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eae00/0x36f9bdc9e421c8e9 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xedd:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fc052 expref: -99 pid: 21885 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00020000:00200000:2.0:1775764845.757093:0:21885:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00100000:2.0:1775764845.757181:0:2381:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1775764845.757197:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967039744:192.168.202.146@tcp:2:touch.0 00000100:00100000:3.0:1775764845.766823:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967039744:192.168.202.146@tcp:2:touch.0 00000080:00200000:0.0:1775764845.767033:0:21885:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xedd:0x0] ignore/verify layout 0/0, layout version 1 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:0.0:1775764845.767081:0:21885:0:(llite_lib.c:2524:ll_setattr_raw()) COMPLETED file file1:[0x200000401:0xedd:0x0](ffff8f9952eb1988) current size 0, valid attrs 201f0, mode 8f99, uid -1048412592, gid -20550, new size 18446744071971469847, atime 1775764845.000000000, mtime 1775764845.000000000, ctime 1775764845.000000000, rc 0 00000080:00200008:0.0:1775764845.767117:0:21885:0:(file.c:447:ll_file_release()) START file file1:[0x200000401:0xedd:0x0](ffff8f9952eb1988), flags 104001 00000002:00100000:0.0:1775764845.767181:0:21885:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f99703d3100 x1862022967039232/t8589934601(8589934601) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 656/744 e 0 to 0 dl 1775764861 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1775764845.767193:0:21885:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1775764845.767198:0:21885:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970a74380 x1862022967040000/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/736 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1775764845.767222:0:21885:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job touch:3f83546c-01a9-4156-8a97-44c446245d3e:21885:1862022967040000:192.168.202.146@tcp:35:touch.0 00000100:00100000:0.0:1775764845.767422:0:21885:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f996687dcc0 going to sleep for 11 seconds 00000100:00100000:3.0:1775764845.775198:0:21885:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job touch:3f83546c-01a9-4156-8a97-44c446245d3e:21885:1862022967040000:192.168.202.146@tcp:35:touch.0 00000002:00100000:3.0:1775764845.775230:0:21885:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f99703d3100 x1862022967039232/t8589934601(8589934601) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 656/744 e 0 to 0 dl 1775764861 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1775764845.821675:0:21888:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:0.0:1775764845.821703:0:21888:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764845.821721:0:21888:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764845.829464:0:21888:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1775764845.829490:0:21888:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764845.829503:0:21888:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:0.0:1775764845.829507:0:21888:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c8c6 00000080:00010000:0.0:1775764845.829517:0:21888:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c8c6 from it ffffafbac19bfbf8 00010000:00010000:0.0:1775764845.829521:0:21888:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764845.829532:0:21888:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:0.0:1775764845.829585:0:21888:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:0.0:1775764845.829596:0:21888:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d10.sanity-pfl, flags=27 00000080:00200000:1.0:1775764845.829825:0:21888:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedc:0x0](ffff8f9952e49148), inode mode 41ed mask 1 00000080:00200000:1.0:1775764845.829846:0:21888:0:(namei.c:1399:ll_atomic_open()) VFS Op:name=file1, dir=[0x200000401:0xedc:0x0](ffff8f9952e49148), file ffff8f9943733b00, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1775764845.829853:0:21888:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=file1, dir=[0x200000401:0xedc:0x0](ffff8f9952e49148), intent=open 00000002:00010000:1.0:1775764845.829884:0:21888:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: file1,[0x0:0x0:0x0]) in obj [0x200000401:0xedc:0x0], intent: open flags 0304001 00010000:00010000:1.0:1775764845.829962:0:21888:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c8f0 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedc:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21888 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764845.829970:0:21888:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c8f0 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedc:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21888 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764845.830001:0:21888:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764845.830015:0:21888:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e6300 x1862022967040256/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 760/131968 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764845.830025:0:21888:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c8f0 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedc:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21888 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764845.830048:0:21888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21888:1862022967040256:192.168.202.146@tcp:101:lfs.0 00000100:00100000:1.0:1775764845.830235:0:21888:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413000 going to sleep for 11 seconds 00000100:00100000:1.0:1775764845.841962:0:21888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21888:1862022967040256:192.168.202.146@tcp:101:lfs.0 00010000:00010000:1.0:1775764845.841986:0:21888:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c8f0 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedc:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21888 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764845.842001:0:21888:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c8f0 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedc:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21888 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764845.842008:0:21888:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c8f0 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedc:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21888 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764845.842015:0:21888:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c8f0 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xedc:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21888 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764845.842023:0:21888:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c8f0 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedc:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21888 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764845.842029:0:21888:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970641600 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:1.0:1775764845.842034:0:21888:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c8f0 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedc:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21888 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764845.842085:0:21888:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c8f0 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedc:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21888 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764845.842092:0:21888:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c8f0 lrc: 5/0,0 mode: --/CR res: [0x200000401:0xedc:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21888 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764845.842098:0:21888:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1775764845.842105:0:21888:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c8f0 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xedc:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21888 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764845.842111:0:21888:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c8f0 lrc: 0/0,0 mode: --/CR res: [0x200000401:0xedc:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21888 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764845.842130:0:21888:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff8f994f2e6300 x1862022967040256/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 760/608 e 0 to 0 dl 1775764861 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764845.842144:0:21888:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0xedc:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1775764845.842302:0:21888:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764845.842324:0:21888:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764845.842337:0:21888:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764845.842348:0:21888:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764845.842355:0:21888:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764845.842361:0:21888:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764845.842364:0:21888:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c8c6 00000080:00010000:1.0:1775764845.842368:0:21888:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c8c6 from it ffffafbac19bfbf8 00010000:00010000:1.0:1775764845.842385:0:21888:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764845.842392:0:21888:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764845.842406:0:21888:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764845.842414:0:21888:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d10.sanity-pfl, flags=267 00000080:00200000:1.0:1775764845.842421:0:21888:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedc:0x0](ffff8f9952e49148), inode mode 41ed mask 44 00000080:00200008:1.0:1775764845.842429:0:21888:0:(file.c:1057:ll_file_open()) START file d10.sanity-pfl:[0x200000401:0xedc:0x0](ffff8f9952e49148), flags 304000 00000002:00010000:1.0:1775764845.842447:0:21888:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xedc:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100240304001 00010000:00010000:1.0:1775764845.842488:0:21888:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641000/0x36f9bdc9e421c8f7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21888 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764845.842495:0:21888:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641000/0x36f9bdc9e421c8f7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21888 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764845.842505:0:21888:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764845.842511:0:21888:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e6300 x1862022967040512/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 760/131968 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764845.842520:0:21888:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641000/0x36f9bdc9e421c8f7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21888 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764845.842539:0:21888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21888:1862022967040512:192.168.202.146@tcp:101:lfs.0 00000100:00100000:1.0:1775764845.842711:0:21888:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413000 going to sleep for 11 seconds 00000100:00100000:1.0:1775764845.846279:0:21888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21888:1862022967040512:192.168.202.146@tcp:101:lfs.0 00010000:00010000:1.0:1775764845.846305:0:21888:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641000/0x36f9bdc9e421c8f7 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21888 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764845.846345:0:21888:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641000/0x36f9bdc9e421c8f7 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21888 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764845.846353:0:21888:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641000/0x36f9bdc9e421c8f7 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21888 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764845.846361:0:21888:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641000/0x36f9bdc9e421c8f7 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 21888 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764845.846369:0:21888:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641000/0x36f9bdc9e421c8f7 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 21888 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764845.846376:0:21888:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970641000 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:1.0:1775764845.846381:0:21888:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641000/0x36f9bdc9e421c8f7 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21888 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764845.846388:0:21888:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641000/0x36f9bdc9e421c8f7 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 21888 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764845.846396:0:21888:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641000/0x36f9bdc9e421c8f7 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 21888 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764845.846403:0:21888:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1775764845.846409:0:21888:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641000/0x36f9bdc9e421c8f7 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21888 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764845.846416:0:21888:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641000/0x36f9bdc9e421c8f7 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 21888 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764845.846436:0:21888:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8f994f2e6300 x1862022967040512/t8589934604(8589934604) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 760/608 e 0 to 0 dl 1775764861 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764845.846469:0:21888:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f994f2e6300 x1862022967040512/t8589934604(8589934604) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 760/608 e 0 to 0 dl 1775764861 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764845.846482:0:21888:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1775764845.846506:0:21888:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0xedc:0x0], updating i_size 512 00000002:00100000:1.0:1775764845.846526:0:21888:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f994f2e6300 x1862022967040512/t8589934604(8589934604) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 760/608 e 0 to 0 dl 1775764861 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:1.0:1775764845.846633:0:21888:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000401:0xedc:0x0](ffff8f9952e49148) cmd=c0086915 arg=1fd23d0 00000100:00100000:1.0:1775764845.846706:0:21888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5500 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21888:1862022967040640:192.168.202.146@tcp:34:lfs.0 00000100:00100000:1.0:1775764845.846880:0:21888:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413000 going to sleep for 11 seconds 00000100:00100000:1.0:1775764845.871612:0:21888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5500 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21888:1862022967040640:192.168.202.146@tcp:34:lfs.0 00000080:00200008:1.0:1775764845.871872:0:21888:0:(file.c:447:ll_file_release()) START file d10.sanity-pfl:[0x200000401:0xedc:0x0](ffff8f9952e49148), flags 304000 00000002:00100000:1.0:1775764845.871944:0:21888:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f994f2e6300 x1862022967040512/t8589934604(8589934604) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 760/608 e 0 to 0 dl 1775764861 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1775764845.871955:0:21888:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764845.871960:0:21888:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e5500 x1862022967040896/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/736 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764845.871981:0:21888:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5500 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21888:1862022967040896:192.168.202.146@tcp:35:lfs.0 00000100:00100000:1.0:1775764845.872063:0:21888:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413000 going to sleep for 11 seconds 00000100:00100000:1.0:1775764845.879401:0:21888:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5500 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21888:1862022967040896:192.168.202.146@tcp:35:lfs.0 00000002:00100000:1.0:1775764845.879438:0:21888:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f994f2e6300 x1862022967040512/t8589934604(8589934604) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 760/608 e 0 to 0 dl 1775764861 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764845.920671:0:21890:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764845.920771:0:21890:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764845.920790:0:21890:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764845.920807:0:21890:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764845.920814:0:21890:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764845.920824:0:21890:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764845.920828:0:21890:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c8c6 00000080:00010000:1.0:1775764845.920834:0:21890:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c8c6 from it ffffafbac1827bf8 00010000:00010000:1.0:1775764845.920838:0:21890:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764845.920846:0:21890:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:1.0:1775764845.920861:0:21890:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764845.921507:0:21890:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d10.sanity-pfl, flags=777 00000080:00200000:1.0:1775764845.921547:0:21890:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764845.921559:0:21890:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764845.921571:0:21890:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764845.921581:0:21890:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:1.0:1775764845.921587:0:21890:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764845.921592:0:21890:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764845.921615:0:21890:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c8c6 00000080:00010000:1.0:1775764845.921619:0:21890:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c8c6 from it ffffafbac1827bf8 00010000:00010000:1.0:1775764845.921623:0:21890:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764845.921630:0:21890:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1775764845.921642:0:21890:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764845.921647:0:21890:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d10.sanity-pfl, flags=267 00000080:00200000:1.0:1775764845.921651:0:21890:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedc:0x0](ffff8f9952e49148), inode mode 41ed mask 44 00000080:00200008:1.0:1775764845.921659:0:21890:0:(file.c:1057:ll_file_open()) START file d10.sanity-pfl:[0x200000401:0xedc:0x0](ffff8f9952e49148), flags 320400 00000002:00010000:1.0:1775764845.921675:0:21890:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xedc:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 04240320401 00010000:00010000:1.0:1775764845.921732:0:21890:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641000/0x36f9bdc9e421c8fe lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 21890 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764845.921858:0:21890:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641000/0x36f9bdc9e421c8fe lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21890 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764845.921870:0:21890:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764845.921885:0:21890:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e5500 x1862022967041152/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/131968 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764845.921895:0:21890:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641000/0x36f9bdc9e421c8fe lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 21890 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764845.921919:0:21890:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5500 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21890:1862022967041152:192.168.202.146@tcp:101:lfs.0 00000100:00100000:1.0:1775764845.922180:0:21890:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413000 going to sleep for 11 seconds 00000100:00100000:1.0:1775764845.936719:0:21890:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5500 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21890:1862022967041152:192.168.202.146@tcp:101:lfs.0 00010000:00010000:1.0:1775764845.936820:0:21890:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641000/0x36f9bdc9e421c8fe lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedc:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc098 expref: -99 pid: 21890 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764845.936832:0:21890:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641000/0x36f9bdc9e421c8fe lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedc:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc098 expref: -99 pid: 21890 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764845.936845:0:21890:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641000/0x36f9bdc9e421c8fe lrc: 4/1,0 mode: CR/CR res: [0x200000401:0xedc:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc098 expref: -99 pid: 21890 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764845.936856:0:21890:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641000/0x36f9bdc9e421c8fe lrc: 4/1,0 mode: CR/CR res: [0x200000401:0xedc:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc098 expref: -99 pid: 21890 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764845.936871:0:21890:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=200002b, status=0 req@ffff8f994f2e5500 x1862022967041152/t8589934606(8589934606) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/608 e 0 to 0 dl 1775764861 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764845.936885:0:21890:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f994f2e5500 x1862022967041152/t8589934606(8589934606) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/608 e 0 to 0 dl 1775764861 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764845.936896:0:21890:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641000/0x36f9bdc9e421c8fe lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xedc:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc098 expref: -99 pid: 21890 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764845.936908:0:21890:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 16 res 8589935617/3804 (0 0) 00000080:00200000:1.0:1775764845.936916:0:21890:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1775764845.936949:0:21890:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0xedc:0x0], updating i_size 512 00000080:00010000:1.0:1775764845.936954:0:21890:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xedc:0x0](ffff8f9952e49148) for lock 0x36f9bdc9e421c8fe 00000080:00010000:1.0:1775764845.936962:0:21890:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c8fe from it ffffafbac1827bd0 00010000:00010000:1.0:1775764845.936966:0:21890:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641000/0x36f9bdc9e421c8fe lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xedc:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc098 expref: -99 pid: 21890 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764845.936995:0:21890:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641000/0x36f9bdc9e421c8fe lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xedc:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc098 expref: -99 pid: 21890 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000002:00100000:1.0:1775764845.937023:0:21890:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f994f2e5500 x1862022967041152/t8589934606(8589934606) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/608 e 0 to 0 dl 1775764861 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:1.0:1775764845.937115:0:21890:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000401:0xedc:0x0](ffff8f9952e49148) cmd=4008669a arg=16612e0 00000020:00100000:1.0:1775764845.937167:0:21890:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1775764845.937174:0:21890:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e6300 x1862022967041408/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764845.937195:0:21890:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21890:1862022967041408:192.168.202.146@tcp:36:lfs.0 00000100:00100000:1.0:1775764845.937266:0:21890:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413000 going to sleep for 11 seconds 00000100:00100000:1.0:1775764845.953179:0:21890:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21890:1862022967041408:192.168.202.146@tcp:36:lfs.0 00000080:00200008:1.0:1775764845.953243:0:21890:0:(file.c:447:ll_file_release()) START file d10.sanity-pfl:[0x200000401:0xedc:0x0](ffff8f9952e49148), flags 320000 00010000:00010000:1.0:1775764845.953288:0:21890:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641000/0x36f9bdc9e421c8fe lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xedc:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc098 expref: -99 pid: 21890 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764845.971126:0:21891:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764845.971153:0:21891:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764845.971172:0:21891:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764845.971188:0:21891:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1775764845.971196:0:21891:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764845.971228:0:21891:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764845.971231:0:21891:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c8c6 00000080:00010000:1.0:1775764845.971238:0:21891:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c8c6 from it ffffafbac1d73bf8 00010000:00010000:1.0:1775764845.971242:0:21891:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764845.971995:0:21891:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1775764845.972018:0:21891:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764845.972028:0:21891:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d10.sanity-pfl, flags=25 00000080:00200000:1.0:1775764845.972036:0:21891:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedc:0x0](ffff8f9952e49148), inode mode 41ed mask 1 00000080:00200000:1.0:1775764845.972053:0:21891:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedc:0x0](ffff8f9952e49148), inode mode 41ed mask 3 00000080:00200000:1.0:1775764845.972058:0:21891:0:(namei.c:1399:ll_atomic_open()) VFS Op:name=file2, dir=[0x200000401:0xedc:0x0](ffff8f9952e49148), file ffff8f99439cb680, open_flags 8941, mode 81b6 opened 0 00000080:00200000:1.0:1775764845.972066:0:21891:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=file2, dir=[0x200000401:0xedc:0x0](ffff8f9952e49148), intent=open|creat 00000002:00010000:1.0:1775764845.972088:0:21891:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: file2,[0x200000401:0xede:0x0]) in obj [0x200000401:0xedc:0x0], intent: open|creat flags 0104502 00010000:00010000:1.0:1775764845.972147:0:21891:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c905 lrc: 3/0,1 mode: --/CW res: [0x200000401:0xedc:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 21891 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764845.972154:0:21891:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c905 lrc: 3/0,1 mode: --/CW res: [0x200000401:0xedc:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 21891 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764845.972167:0:21891:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764845.972178:0:21891:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e7b80 x1862022967041664/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 760/66432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764845.972187:0:21891:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c905 lrc: 3/0,1 mode: --/CW res: [0x200000401:0xedc:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 21891 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764845.972209:0:21891:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job touch:3f83546c-01a9-4156-8a97-44c446245d3e:21891:1862022967041664:192.168.202.146@tcp:101:touch.0 00000100:00100000:0.0:1775764845.980656:0:21891:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952413000 going to sleep for 11 seconds 00000100:00100000:3.0:1775764845.992383:0:21891:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job touch:3f83546c-01a9-4156-8a97-44c446245d3e:21891:1862022967041664:192.168.202.146@tcp:101:touch.0 00010000:00010000:3.0:1775764845.992414:0:21891:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c905 lrc: 4/0,1 mode: --/CW res: [0x200000401:0xedc:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc0c2 expref: -99 pid: 21891 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764845.992450:0:21891:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c905 lrc: 4/0,1 mode: --/PR res: [0x200000401:0xede:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc0c2 expref: -99 pid: 21891 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764845.992458:0:21891:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c905 lrc: 4/0,1 mode: --/PR res: [0x200000401:0xede:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc0c2 expref: -99 pid: 21891 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764845.992470:0:21891:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c905 lrc: 4/0,1 mode: PR/PR res: [0x200000401:0xede:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc0c2 expref: -99 pid: 21891 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764845.992481:0:21891:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c905 lrc: 4/0,1 mode: PR/PR res: [0x200000401:0xede:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc0c2 expref: -99 pid: 21891 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764845.992494:0:21891:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c905 lrc: 5/1,1 mode: PR/PR res: [0x200000401:0xede:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc0c2 expref: -99 pid: 21891 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764845.992502:0:21891:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c905 lrc: 5/1,1 mode: PR/PR res: [0x200000401:0xede:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc0c2 expref: -99 pid: 21891 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764845.992557:0:21891:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c905 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xede:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc0c2 expref: -99 pid: 21891 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764845.992569:0:21891:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=3 disposition=37, status=0 req@ffff8f994f2e7b80 x1862022967041664/t8589934608(8589934608) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 760/952 e 0 to 0 dl 1775764861 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000002:00100000:3.0:1775764845.992581:0:21891:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f994f2e7b80 x1862022967041664/t8589934608(8589934608) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 760/952 e 0 to 0 dl 1775764861 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1775764845.992595:0:21891:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c905 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xede:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc0c2 expref: -99 pid: 21891 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:3.0:1775764845.992608:0:21891:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c905 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xede:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc0c2 expref: -99 pid: 21891 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764845.992619:0:21891:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/3806 (0 0) 00000080:00200000:3.0:1775764845.992626:0:21891:0:(namei.c:943:ll_intent_lock()) intent lock 3 on i1 [0x200000401:0xedc:0x0] suppgids 0 -1: rc 0 00000080:00200000:3.0:1775764845.992642:0:21891:0:(namei.c:1610:ll_create_it()) VFS Op:name=file2, dir=[0x200000401:0xedc:0x0](ffff8f9952e49148), intent=open|creat 00000080:00200000:3.0:1775764845.992683:0:21891:0:(llite_lib.c:3177:ll_read_inode2()) VFS Op:inode=[0x200000401:0xede:0x0](ffff8f9952eeaa08) 00000080:00200000:3.0:1775764845.992902:0:21891:0:(file.c:6766:ll_layout_conf()) [0x200000401:0xede:0x0]: layout version change: 4294967294 -> 3 00000080:00010000:3.0:1775764845.992913:0:21891:0:(namei.c:1576:ll_create_node()) setting l_ast_data to inode [0x200000401:0xede:0x0](ffff8f9952eeaa08) 00000080:00010000:3.0:1775764845.992916:0:21891:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xede:0x0](ffff8f9952eeaa08) for lock 0x36f9bdc9e421c905 00000080:00010000:3.0:1775764845.992954:0:21891:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xede:0x0](ffff8f9952eeaa08) 00000080:00200008:3.0:1775764845.992969:0:21891:0:(file.c:1057:ll_file_open()) START file file2:[0x200000401:0xede:0x0](ffff8f9952eeaa08), flags 104501 00000002:00100000:3.0:1775764845.992988:0:21891:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f994f2e7b80 x1862022967041664/t8589934608(8589934608) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/952 e 0 to 0 dl 1775764861 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000080:00010000:3.0:1775764845.993034:0:21891:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c905 from it ffff8f994ed57240 00010000:00010000:3.0:1775764845.993052:0:21891:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c905 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xede:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fc0c2 expref: -99 pid: 21891 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764845.993061:0:21891:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c905 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xede:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fc0c2 expref: -99 pid: 21891 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764845.993087:0:21891:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xede:0x0](ffff8f9952eeaa08), inode mode 81a4 mask 40 00000080:00200008:3.0:1775764845.993126:0:21891:0:(llite_lib.c:2274:ll_setattr_raw()) START file file2:[0x200000401:0xede:0x0](ffff8f9952eeaa08) current size 0, valid attrs 20070, mode 8f99, uid -1042858416, gid -20550, new size 18446744071971469847, atime 1775764845.000000000, mtime 1775764845.000000000, ctime 1775764845.000000000 00010000:00010000:3.0:1775764845.993147:0:21891:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c905 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xede:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5fc0c2 expref: -99 pid: 21891 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764845.993170:0:21891:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0xede:0x0]: losing layout lock 00010000:00010000:3.0:1775764845.993213:0:21891:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c905 lrc: 1/0,0 mode: --/PR res: [0x200000401:0xede:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fc0c2 expref: -99 pid: 21891 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764845.993220:0:21891:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764845.993222:0:21891:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c905 lrc: 0/0,0 mode: --/PR res: [0x200000401:0xede:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fc0c2 expref: -99 pid: 21891 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000020:00100000:3.0:1775764845.993245:0:21891:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1775764845.993251:0:21891:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970edc700 x1862022967041920/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764845.993273:0:21891:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job touch:3f83546c-01a9-4156-8a97-44c446245d3e:21891:1862022967041920:192.168.202.146@tcp:36:touch.0 00000100:00100000:3.0:1775764845.993353:0:21891:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560d80 going to sleep for 11 seconds 00000100:00100000:3.0:1775764846.006461:0:21891:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job touch:3f83546c-01a9-4156-8a97-44c446245d3e:21891:1862022967041920:192.168.202.146@tcp:36:touch.0 00000080:00200000:3.0:1775764846.006545:0:21891:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xede:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00010000:3.0:1775764846.006611:0:21891:0:(file.c:6988:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200000401:0xede:0x0](ffff8f9952eeaa08) 00000002:00010000:3.0:1775764846.006622:0:21891:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xede:0x0]) in obj [0x200000401:0xede:0x0], intent: layout flags 0 00010000:00010000:3.0:1775764846.006677:0:21891:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c90c lrc: 3/1,0 mode: --/CR res: [0x200000401:0xede:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 21891 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764846.006687:0:21891:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c90c lrc: 3/1,0 mode: --/CR res: [0x200000401:0xede:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21891 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764846.006697:0:21891:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c90c lrc: 3/1,0 mode: --/CR res: [0x200000401:0xede:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 21891 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764846.006716:0:21891:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job touch:3f83546c-01a9-4156-8a97-44c446245d3e:21891:1862022967042048:192.168.202.146@tcp:101:touch.0 00000100:00100000:3.0:1775764846.006821:0:21891:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560d80 going to sleep for 11 seconds 00000100:00100000:3.0:1775764846.025210:0:21891:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job touch:3f83546c-01a9-4156-8a97-44c446245d3e:21891:1862022967042048:192.168.202.146@tcp:101:touch.0 00010000:00010000:3.0:1775764846.025243:0:21891:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c90c lrc: 4/1,0 mode: CR/CR res: [0x200000401:0xede:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc0d7 expref: -99 pid: 21891 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764846.025257:0:21891:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c90c lrc: 4/1,0 mode: CR/CR res: [0x200000401:0xede:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc0d7 expref: -99 pid: 21891 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764846.025271:0:21891:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8f9970edce00 x1862022967042048/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 376/688 e 0 to 0 dl 1775764862 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'touch.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1775764846.025282:0:21891:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 344 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c90c lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xede:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc0d7 expref: -99 pid: 21891 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:3.0:1775764846.025294:0:21891:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c90c lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xede:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc0d7 expref: -99 pid: 21891 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764846.025331:0:21891:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 16 res 8589935617/3806 (0 0) 00000080:00200000:3.0:1775764846.025337:0:21891:0:(namei.c:943:ll_intent_lock()) intent lock 1024 on i1 [0x200000401:0xede:0x0] suppgids 0 0: rc 0 00000080:00010000:3.0:1775764846.029245:0:21891:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xede:0x0](ffff8f9952eeaa08) for lock 0x36f9bdc9e421c90c 00000080:00010000:3.0:1775764846.029262:0:21891:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c90c from it ffffafbac1d73a78 00010000:00010000:3.0:1775764846.029269:0:21891:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c90c lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xede:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc0d7 expref: -99 pid: 21891 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764846.029283:0:21891:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c90c lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xede:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc0d7 expref: -99 pid: 21891 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764846.029332:0:21891:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c90c lrc: 2/1,0 mode: CR/CR res: [0x200000401:0xede:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc0d7 expref: -99 pid: 21891 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764846.029357:0:21891:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c90c lrc: 2/1,0 mode: CR/CR res: [0x200000401:0xede:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc0d7 expref: -99 pid: 21891 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:3.0:1775764846.029368:0:21891:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200000401:0xede:0x0](000000000ba2c740) being reconfigured ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c90c lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xede:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc0d7 expref: -99 pid: 21891 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764846.029612:0:21891:0:(file.c:6766:ll_layout_conf()) [0x200000401:0xede:0x0]: layout version change: 4294967294 -> 3 00010000:00010000:3.0:1775764846.029632:0:21891:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c90c lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xede:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fc0d7 expref: -99 pid: 21891 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764846.029640:0:21891:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5400/0x36f9bdc9e421c90c lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xede:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fc0d7 expref: -99 pid: 21891 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00020000:00200000:3.0:1775764846.029660:0:21891:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764846.029698:0:21891:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:3.0:1775764846.029700:0:21891:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000100:00100000:3.0:1775764846.029800:0:2382:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1775764846.029818:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967042176:192.168.202.146@tcp:2:touch.0 00000100:00100000:3.0:1775764846.034694:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967042176:192.168.202.146@tcp:2:touch.0 00000080:00200000:0.0:1775764846.034820:0:21891:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xede:0x0] ignore/verify layout 0/0, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:0.0:1775764846.034841:0:21891:0:(llite_lib.c:2524:ll_setattr_raw()) COMPLETED file file2:[0x200000401:0xede:0x0](ffff8f9952eeaa08) current size 0, valid attrs 201f0, mode 8f99, uid -1042858416, gid -20550, new size 18446744071971469847, atime 1775764845.000000000, mtime 1775764845.000000000, ctime 1775764845.000000000, rc 0 00000080:00200008:0.0:1775764846.034883:0:21891:0:(file.c:447:ll_file_release()) START file file2:[0x200000401:0xede:0x0](ffff8f9952eeaa08), flags 104001 00000002:00100000:0.0:1775764846.034950:0:21891:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f994f2e7b80 x1862022967041664/t8589934608(8589934608) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/952 e 0 to 0 dl 1775764861 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1775764846.034962:0:21891:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1775764846.034967:0:21891:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970a75180 x1862022967042432/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/840 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1775764846.034991:0:21891:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job touch:3f83546c-01a9-4156-8a97-44c446245d3e:21891:1862022967042432:192.168.202.146@tcp:35:touch.0 00000100:00100000:0.0:1775764846.035072:0:21891:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f996687df00 going to sleep for 11 seconds 00000100:00100000:0.0:1775764846.048655:0:21891:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job touch:3f83546c-01a9-4156-8a97-44c446245d3e:21891:1862022967042432:192.168.202.146@tcp:35:touch.0 00000002:00100000:0.0:1775764846.048691:0:21891:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f994f2e7b80 x1862022967041664/t8589934608(8589934608) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/952 e 0 to 0 dl 1775764861 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'touch.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1775764846.079296:0:21894:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:0.0:1775764846.079322:0:21894:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764846.079340:0:21894:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:0.0:1775764846.079355:0:21894:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:0.0:1775764846.079408:0:21894:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764846.079416:0:21894:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:0.0:1775764846.079419:0:21894:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c8c6 00000080:00010000:0.0:1775764846.079426:0:21894:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c8c6 from it ffffafbac1c43bf8 00010000:00010000:0.0:1775764846.079430:0:21894:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:0.0:1775764846.079440:0:21894:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:0.0:1775764846.079454:0:21894:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:0.0:1775764846.079461:0:21894:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d10.sanity-pfl, flags=27 00000080:00200000:0.0:1775764846.079466:0:21894:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedc:0x0](ffff8f9952e49148), inode mode 41ed mask 1 00000080:00200000:0.0:1775764846.079476:0:21894:0:(namei.c:1399:ll_atomic_open()) VFS Op:name=file2, dir=[0x200000401:0xedc:0x0](ffff8f9952e49148), file ffff8f9952bed980, open_flags 18800, mode 0 opened 0 00000080:00200000:0.0:1775764846.079482:0:21894:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=file2, dir=[0x200000401:0xedc:0x0](ffff8f9952e49148), intent=open 00000002:00010000:0.0:1775764846.079494:0:21894:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: file2,[0x0:0x0:0x0]) in obj [0x200000401:0xedc:0x0], intent: open flags 0304001 00010000:00010000:0.0:1775764846.079542:0:21894:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c913 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedc:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 21894 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764846.079549:0:21894:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c913 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 21894 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1775764846.079561:0:21894:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1775764846.079573:0:21894:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970a76d80 x1862022967042688/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/132072 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1775764846.079582:0:21894:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c913 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 21894 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764846.079617:0:21894:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21894:1862022967042688:192.168.202.146@tcp:101:lfs.0 00000100:00100000:0.0:1775764846.079825:0:21894:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f996687df00 going to sleep for 11 seconds 00000100:00100000:0.0:1775764846.091224:0:21894:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21894:1862022967042688:192.168.202.146@tcp:101:lfs.0 00010000:00010000:0.0:1775764846.091248:0:21894:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c913 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 21894 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764846.091262:0:21894:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c913 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 21894 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764846.091268:0:21894:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c913 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 21894 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764846.091274:0:21894:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c913 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xedc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 21894 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764846.091282:0:21894:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c913 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 21894 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764846.091287:0:21894:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99821cce00 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:0.0:1775764846.091292:0:21894:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c913 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedc:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 21894 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764846.091298:0:21894:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c913 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedc:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 21894 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764846.091305:0:21894:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c913 lrc: 5/0,0 mode: --/CR res: [0x200000401:0xedc:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 21894 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764846.091311:0:21894:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1775764846.091337:0:21894:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c913 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xedc:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 21894 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764846.091344:0:21894:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c913 lrc: 0/0,0 mode: --/CR res: [0x200000401:0xedc:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 21894 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764846.091374:0:21894:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff8f9970a76d80 x1862022967042688/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/608 e 0 to 0 dl 1775764862 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1775764846.091389:0:21894:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0xedc:0x0] suppgids 0 -1: rc -20 00000080:00200000:0.0:1775764846.091554:0:21894:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:0.0:1775764846.091575:0:21894:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764846.091591:0:21894:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:0.0:1775764846.091603:0:21894:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:0.0:1775764846.091610:0:21894:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764846.091615:0:21894:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:0.0:1775764846.091619:0:21894:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c8c6 00000080:00010000:0.0:1775764846.091624:0:21894:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c8c6 from it ffffafbac1c43bf8 00010000:00010000:0.0:1775764846.091627:0:21894:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:0.0:1775764846.091634:0:21894:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:0.0:1775764846.091647:0:21894:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:0.0:1775764846.091655:0:21894:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d10.sanity-pfl, flags=267 00000080:00200000:0.0:1775764846.091678:0:21894:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedc:0x0](ffff8f9952e49148), inode mode 41ed mask 44 00000080:00200008:0.0:1775764846.091687:0:21894:0:(file.c:1057:ll_file_open()) START file d10.sanity-pfl:[0x200000401:0xedc:0x0](ffff8f9952e49148), flags 304000 00000080:00200080:0.0:1775764846.091722:0:21894:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000401:0xedc:0x0](ffff8f9952e49148) cmd=c0086915 arg=1db93d0 00000100:00100000:0.0:1775764846.091814:0:21894:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21894:1862022967042816:192.168.202.146@tcp:34:lfs.0 00000100:00100000:0.0:1775764846.091891:0:21894:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f996687df00 going to sleep for 11 seconds 00000100:00100000:0.0:1775764846.108987:0:21894:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:21894:1862022967042816:192.168.202.146@tcp:34:lfs.0 00000080:00200008:0.0:1775764846.109264:0:21894:0:(file.c:447:ll_file_release()) START file d10.sanity-pfl:[0x200000401:0xedc:0x0](ffff8f9952e49148), flags 304000 00010000:00010000:0.0:1775764846.109295:0:21894:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641000/0x36f9bdc9e421c8fe lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xedc:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc098 expref: -99 pid: 21890 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764846.153907:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764847.176694:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764847.301983:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967043072:192.168.202.146@tcp:400:kworker.0 00000100:00100000:1.0:1775764847.301983:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2380:1862022967042944:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764847.302100:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967043328:192.168.202.146@tcp:400:kworker.0 00000100:00100000:1.0:1775764847.302101:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967043200:192.168.202.146@tcp:400:kworker.0 00000100:00100000:1.0:1775764847.310476:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967043072:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764847.316319:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2380:1862022967042944:192.168.202.146@tcp:400:kworker.0 00000100:00100000:1.0:1775764847.322421:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967043328:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764847.322833:0:2380:0:(client.c:3082:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8f99523ee800: committing for last_committed 4294971565 gen 5 00000100:00100000:0.0:1775764847.322848:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967043200:192.168.202.146@tcp:400:kworker.0 00010000:00010000:1.0:1775764848.199546:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764849.225027:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764850.245901:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764851.273146:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:3.0:1775764851.591968:0:21728:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8f9952d121c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764851.591980:0:21728:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:3.0:1775764851.592004:0:21728:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764851.592024:0:21728:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:3.0:1775764851.592039:0:21728:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:3.0:1775764851.592046:0:21728:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764851.592054:0:21728:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:3.0:1775764851.592057:0:21728:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c8c6 00000080:00010000:3.0:1775764851.592063:0:21728:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c8c6 from it ffffafbac182fcb8 00010000:00010000:3.0:1775764851.592067:0:21728:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:3.0:1775764851.592076:0:21728:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00000080:00200000:2.0:1775764851.605265:0:22310:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:2.0:1775764851.605298:0:22310:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764851.605319:0:22310:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00010000:00010000:2.0:1775764851.605336:0:22310:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00000080:00200000:2.0:1775764851.605344:0:22310:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764851.605517:0:22310:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:2.0:1775764851.605522:0:22310:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c8c6 00000080:00010000:2.0:1775764851.605530:0:22310:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c8c6 from it ffffafbac1097bb0 00010000:00010000:2.0:1775764851.605533:0:22310:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00010000:00010000:2.0:1775764851.605627:0:22310:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:2.0:1775764851.605646:0:22310:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 44 00000080:00200008:2.0:1775764851.605656:0:22310:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8f9952d121c8), flags 320400 00000080:00200080:2.0:1775764851.605677:0:22310:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8) cmd=4008669a arg=24312e0 00010000:00010000:2.0:1775764851.605697:0:22310:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:2.0:1775764851.605774:0:22310:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:2.0:1775764851.605786:0:22310:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764851.605788:0:22310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421c8c6 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fbfd4 expref: -99 pid: 21884 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000020:00100000:2.0:1775764851.605811:0:22310:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1775764851.605826:0:22310:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f99703d1180 x1862022967043584/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764851.605856:0:22310:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22310:1862022967043584:192.168.202.146@tcp:36:lfs.0 00000100:00100000:2.0:1775764851.605982:0:22310:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952af3600 going to sleep for 11 seconds 00000100:00100000:3.0:1775764851.614442:0:22310:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8f99523ee800: committing for last_committed 8589934610 gen 1 00000100:00100000:3.0:1775764851.614485:0:22310:0:(client.c:3131:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff8f994f2e5500 x1862022967041152/t8589934606(8589934606) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/608 e 0 to 0 dl 1775764861 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:3.0:1775764851.614508:0:22310:0:(client.c:3126:ptlrpc_free_committed()) @@@ stopping search req@ffff8f99703d1180 x1862022967043584/t8589934611(8589934611) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/456 e 0 to 0 dl 1775764867 ref 3 fl Rpc:RQU/204/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764851.614548:0:22310:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22310:1862022967043584:192.168.202.146@tcp:36:lfs.0 00000080:00200008:3.0:1775764851.614597:0:22310:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8f9952d121c8), flags 320000 00010000:00010000:1.0:1775764852.294894:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764852.428185:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2382:1862022967043712:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764852.429919:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967043840:192.168.202.146@tcp:400:kworker.0 00000100:00100000:2.0:1775764852.430768:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967043968:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764852.430871:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967044096:192.168.202.146@tcp:400:kworker.0 00000100:00100000:2.0:1775764852.447017:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967043840:192.168.202.146@tcp:400:kworker.0 00000100:00100000:2.0:1775764852.447034:0:2381:0:(client.c:3082:ptlrpc_free_committed()) lustre-OST0001-osc-ffff8f99523ee800: committing for last_committed 4294972361 gen 6 00000100:00100000:2.0:1775764852.447092:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967044096:192.168.202.146@tcp:400:kworker.0 00000100:00100000:2.0:1775764852.466154:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2382:1862022967043712:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764852.466239:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967043968:192.168.202.146@tcp:400:kworker.0 00010000:00010000:2.0:1775764853.322304:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:0.0:1775764853.338983:0:22452:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-pfl test 11a: Verify component instantiation with write/truncate ========================================================== 16:00:51 (1775764851) 00000080:00200000:1.0:1775764853.466530:0:22460:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764853.466562:0:22460:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764853.466671:0:22460:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641800/0x36f9bdc9e421c91a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 22460 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764853.466682:0:22460:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641800/0x36f9bdc9e421c91a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 22460 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764853.466692:0:22460:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641800/0x36f9bdc9e421c91a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 22460 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764853.466723:0:22460:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7100 pname:cluuid:pid:xid:nid:opc:job mkdir:3f83546c-01a9-4156-8a97-44c446245d3e:22460:1862022967044224:192.168.202.146@tcp:101:mkdir.0 00000100:00100000:1.0:1775764853.467558:0:22460:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560c00 going to sleep for 11 seconds 00000100:00100000:3.0:1775764853.483279:0:22460:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8f99523ee800: committing for last_committed 8589934611 gen 1 00000100:00100000:3.0:1775764853.483308:0:22460:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7100 pname:cluuid:pid:xid:nid:opc:job mkdir:3f83546c-01a9-4156-8a97-44c446245d3e:22460:1862022967044224:192.168.202.146@tcp:101:mkdir.0 00010000:00010000:3.0:1775764853.483331:0:22460:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641800/0x36f9bdc9e421c91a lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc116 expref: -99 pid: 22460 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764853.483356:0:22460:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641800/0x36f9bdc9e421c91a lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc116 expref: -99 pid: 22460 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764853.483368:0:22460:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641800/0x36f9bdc9e421c91a lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc116 expref: -99 pid: 22460 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764853.483378:0:22460:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641800/0x36f9bdc9e421c91a lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc116 expref: -99 pid: 22460 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764853.483390:0:22460:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641800/0x36f9bdc9e421c91a lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc116 expref: -99 pid: 22460 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764853.483397:0:22460:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641800/0x36f9bdc9e421c91a lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc116 expref: -99 pid: 22460 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764853.483424:0:22460:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641800/0x36f9bdc9e421c91a lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc116 expref: -99 pid: 22460 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764853.483434:0:22460:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994f2e7100 x1862022967044224/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764869 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'mkdir.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1775764853.483447:0:22460:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641800/0x36f9bdc9e421c91a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc116 expref: -99 pid: 22460 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764853.483463:0:22460:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1775764853.483470:0:22460:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1775764853.483498:0:22460:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 8704 00000080:00010000:3.0:1775764853.483505:0:22460:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c91a 00000080:00010000:3.0:1775764853.483511:0:22460:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:3.0:1775764853.483514:0:22460:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c91a from it ffffafbac0e2bbe0 00010000:00010000:3.0:1775764853.483517:0:22460:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641800/0x36f9bdc9e421c91a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc116 expref: -99 pid: 22460 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764853.483524:0:22460:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641800/0x36f9bdc9e421c91a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc116 expref: -99 pid: 22460 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764853.483541:0:22460:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:3.0:1775764853.483569:0:22460:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=d11a.sanity-pfl, dir=[0x200000007:0x1:0x0](ffff8f9952d121c8), flags=1536 00000080:00200000:3.0:1775764853.483573:0:22460:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:3.0:1775764853.483587:0:22460:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764853.483595:0:22460:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641800/0x36f9bdc9e421c91a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc116 expref: -99 pid: 22460 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764853.483627:0:22460:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641800/0x36f9bdc9e421c91a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc116 expref: -99 pid: 22460 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764853.483634:0:22460:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764853.483639:0:22460:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:3.0:1775764853.483641:0:22460:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c91a 00000080:00010000:3.0:1775764853.483645:0:22460:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c91a from it ffffafbac0e2bce0 00010000:00010000:3.0:1775764853.483648:0:22460:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641800/0x36f9bdc9e421c91a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc116 expref: -99 pid: 22460 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764853.483654:0:22460:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641800/0x36f9bdc9e421c91a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc116 expref: -99 pid: 22460 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764853.483662:0:22460:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 3 00000080:00200000:3.0:1775764853.483668:0:22460:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:3.0:1775764853.483674:0:22460:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764853.483679:0:22460:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641800/0x36f9bdc9e421c91a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc116 expref: -99 pid: 22460 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764853.483686:0:22460:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641800/0x36f9bdc9e421c91a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc116 expref: -99 pid: 22460 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764853.483692:0:22460:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764853.483695:0:22460:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:3.0:1775764853.483697:0:22460:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c91a 00000080:00010000:3.0:1775764853.483700:0:22460:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c91a from it ffffafbac0e2bdc8 00010000:00010000:3.0:1775764853.483716:0:22460:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641800/0x36f9bdc9e421c91a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc116 expref: -99 pid: 22460 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764853.483723:0:22460:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641800/0x36f9bdc9e421c91a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc116 expref: -99 pid: 22460 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764853.483764:0:22460:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 3 00000080:00200000:3.0:1775764853.483769:0:22460:0:(namei.c:2216:do_mkdir()) VFS Op:name=d11a.sanity-pfl, dir=[0x200000007:0x1:0x0](ffff8f9952d121c8) 00010000:00010000:3.0:1775764853.483801:0:22460:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641800/0x36f9bdc9e421c91a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fc116 expref: -99 pid: 22460 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764853.483846:0:22460:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641800/0x36f9bdc9e421c91a lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fc116 expref: -99 pid: 22460 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764853.483863:0:22460:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764853.483865:0:22460:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641800/0x36f9bdc9e421c91a lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fc116 expref: -99 pid: 22460 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:3.0:1775764853.483890:0:22460:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1775764853.483896:0:22460:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970edc700 x1862022967044480/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 560/792 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764853.483918:0:22460:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job mkdir:3f83546c-01a9-4156-8a97-44c446245d3e:22460:1862022967044480:192.168.202.146@tcp:36:mkdir.0 00000100:00100000:3.0:1775764853.483990:0:22460:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994ecbb3c0 going to sleep for 11 seconds 00000100:00100000:3.0:1775764853.493151:0:22460:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job mkdir:3f83546c-01a9-4156-8a97-44c446245d3e:22460:1862022967044480:192.168.202.146@tcp:36:mkdir.0 00000080:00200000:3.0:1775764853.493215:0:22460:0:(llite_lib.c:3177:ll_read_inode2()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88) 00000080:00200000:3.0:1775764853.493224:0:22460:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0xedf:0x0], updating i_size 512 00000080:00200000:1.0:1775764853.497843:0:22462:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764853.497881:0:22462:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764853.497982:0:22462:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 22462 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764853.497993:0:22462:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 22462 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764853.498019:0:22462:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 22462 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764853.498047:0:22462:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:22462:1862022967044608:192.168.202.146@tcp:101:rm.0 00000100:00100000:1.0:1775764853.498268:0:22462:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560cc0 going to sleep for 11 seconds 00000100:00100000:0.0:1775764853.511675:0:22462:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:22462:1862022967044608:192.168.202.146@tcp:101:rm.0 00010000:00010000:0.0:1775764853.511706:0:22462:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.511719:0:22462:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.511802:0:22462:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.511814:0:22462:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.511827:0:22462:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.511835:0:22462:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.511842:0:22462:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764853.511862:0:22462:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994f2e6680 x1862022967044608/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764869 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1775764853.511876:0:22462:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.511888:0:22462:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1775764853.511893:0:22462:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1775764853.511924:0:22462:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 8704 00000080:00010000:0.0:1775764853.511931:0:22462:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:0.0:1775764853.511936:0:22462:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:0.0:1775764853.511940:0:22462:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac1473b58 00010000:00010000:0.0:1775764853.511959:0:22462:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.511968:0:22462:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764853.511993:0:22462:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:0.0:1775764853.512010:0:22462:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=d11a.sanity-pfl, dir=[0x200000007:0x1:0x0](ffff8f9952d121c8), flags=24 00000080:00200000:0.0:1775764853.512015:0:22462:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=d11a.sanity-pfl, dir=[0x200000007:0x1:0x0](ffff8f9952d121c8), intent=0 00000002:00010000:0.0:1775764853.512035:0:22462:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: d11a.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: lookup flags 0 00010000:00010000:0.0:1775764853.512077:0:22462:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c928 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 22462 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.512085:0:22462:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c928 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 22462 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.512094:0:22462:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c928 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 22462 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764853.512116:0:22462:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74e00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:22462:1862022967044736:192.168.202.146@tcp:101:rm.0 00000100:00100000:0.0:1775764853.512197:0:22462:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102000 going to sleep for 11 seconds 00000100:00100000:0.0:1775764853.520609:0:22462:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74e00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:22462:1862022967044736:192.168.202.146@tcp:101:rm.0 00010000:00010000:0.0:1775764853.520638:0:22462:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c928 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc14e expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.520659:0:22462:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c928 lrc: 4/1,0 mode: --/PR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc14e expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.520667:0:22462:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c928 lrc: 4/1,0 mode: --/PR res: [0x200000401:0xedf:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc14e expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.520700:0:22462:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c928 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xedf:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc14e expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.520710:0:22462:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c928 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xedf:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc14e expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.520724:0:22462:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c928 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xedf:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc14e expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.520810:0:22462:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c928 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xedf:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc14e expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.520817:0:22462:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c928 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xedf:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc14e expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764853.520829:0:22462:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=b, status=0 req@ffff8f9970a74e00 x1862022967044736/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/584 e 0 to 0 dl 1775764869 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1775764853.520842:0:22462:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c928 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xedf:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc14e expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.520861:0:22462:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/3807 (0 0) 00000080:00200000:0.0:1775764853.520868:0:22462:0:(namei.c:943:ll_intent_lock()) intent lock 16 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1775764853.520890:0:22462:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0xedf:0x0], updating i_size 512 00000080:00200000:0.0:1775764853.520894:0:22462:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0xedf:0x0](ffff8f9952eedb88): rc = 0 00000080:00010000:0.0:1775764853.520898:0:22462:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xedf:0x0](ffff8f9952eedb88) for lock 0x36f9bdc9e421c928 00000080:00010000:0.0:1775764853.520910:0:22462:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xedf:0x0](ffff8f9952eedb88) 00000080:00010000:0.0:1775764853.520928:0:22462:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c928 from it ffffafbac1473a28 00010000:00010000:0.0:1775764853.520931:0:22462:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c928 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xedf:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc14e expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.520941:0:22462:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc400/0x36f9bdc9e421c928 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xedf:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc14e expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764853.520973:0:22462:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764853.520991:0:22462:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=f11a.sanity-pfl, dir=[0x200000401:0xedf:0x0](ffff8f9952eedb88), flags=8 00000080:00200000:0.0:1775764853.520996:0:22462:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=f11a.sanity-pfl, dir=[0x200000401:0xedf:0x0](ffff8f9952eedb88), intent=getattr 00000002:00010000:0.0:1775764853.521013:0:22462:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f11a.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000401:0xedf:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764853.521053:0:22462:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c92f lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 22462 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.521060:0:22462:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c92f lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 22462 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.521070:0:22462:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c92f lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 22462 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764853.521110:0:22462:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74e00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:22462:1862022967044864:192.168.202.146@tcp:101:rm.0 00000100:00100000:0.0:1775764853.521183:0:22462:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102000 going to sleep for 11 seconds 00000100:00100000:2.0:1775764853.553627:0:22462:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74e00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:22462:1862022967044864:192.168.202.146@tcp:101:rm.0 00010000:00010000:2.0:1775764853.553649:0:22462:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c92f lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 22462 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.553668:0:22462:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c92f lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 22462 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.553681:0:22462:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c92f lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 22462 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.553688:0:22462:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c92f lrc: 3/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 22462 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.553696:0:22462:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c92f lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 22462 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.553702:0:22462:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99821ccc00 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:2.0:1775764853.553707:0:22462:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c92f lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 22462 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.553713:0:22462:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c92f lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 22462 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.553721:0:22462:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c92f lrc: 5/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 22462 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.553765:0:22462:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1775764853.553771:0:22462:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c92f lrc: 3/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 22462 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.553779:0:22462:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c92f lrc: 0/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 22462 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764853.553794:0:22462:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=7, status=0 req@ffff8f9970a74e00 x1862022967044864/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/584 e 0 to 0 dl 1775764869 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764853.553808:0:22462:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0xedf:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1775764853.553828:0:22462:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/3807 (0 0) 00000080:00200000:2.0:1775764853.553898:0:22462:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:3.0:1775764853.553955:0:22462:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764853.553987:0:22462:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764853.553998:0:22462:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764853.554005:0:22462:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764853.554012:0:22462:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:3.0:1775764853.554015:0:22462:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:3.0:1775764853.554021:0:22462:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac1473c10 00010000:00010000:3.0:1775764853.554025:0:22462:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764853.554033:0:22462:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764853.554046:0:22462:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:3.0:1775764853.554052:0:22462:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d11a.sanity-pfl, flags=24 00000080:00200000:3.0:1775764853.554057:0:22462:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764853.554069:0:22462:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=f11a.sanity-pfl, dir=[0x200000401:0xedf:0x0](ffff8f9952eedb88), flags=0 00000080:00200000:3.0:1775764853.554074:0:22462:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=f11a.sanity-pfl, dir=[0x200000401:0xedf:0x0](ffff8f9952eedb88), intent=getattr 00000002:00010000:3.0:1775764853.554086:0:22462:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f11a.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000401:0xedf:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764853.554121:0:22462:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c936 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 22462 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764853.554128:0:22462:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c936 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 22462 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764853.554135:0:22462:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c936 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 22462 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764853.554190:0:22462:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:22462:1862022967044992:192.168.202.146@tcp:101:rm.0 00000100:00100000:3.0:1775764853.554276:0:22462:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994ecbb3c0 going to sleep for 11 seconds 00000100:00100000:0.0:1775764853.573424:0:22462:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:22462:1862022967044992:192.168.202.146@tcp:101:rm.0 00010000:00010000:0.0:1775764853.573447:0:22462:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c936 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 22462 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.573461:0:22462:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c936 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 22462 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.573513:0:22462:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c936 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 22462 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.573520:0:22462:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c936 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 22462 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.573528:0:22462:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c936 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 22462 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.573534:0:22462:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99435d4c00 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:0.0:1775764853.573538:0:22462:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c936 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 22462 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.573544:0:22462:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c936 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 22462 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.573551:0:22462:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c936 lrc: 5/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 22462 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.573558:0:22462:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1775764853.573564:0:22462:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c936 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 22462 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.573569:0:22462:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4c00/0x36f9bdc9e421c936 lrc: 0/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 22462 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764853.573586:0:22462:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=7, status=0 req@ffff8f9970ede680 x1862022967044992/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/584 e 0 to 0 dl 1775764869 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1775764853.573599:0:22462:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0xedf:0x0] suppgids 0 -1: rc 0 00010000:00010000:0.0:1775764853.573622:0:22462:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/3807 (0 0) 00000080:00200000:2.0:1775764853.584940:0:22469:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:2.0:1775764853.584968:0:22469:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764853.584988:0:22469:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764853.585003:0:22469:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764853.585012:0:22469:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764853.585020:0:22469:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:2.0:1775764853.585024:0:22469:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:2.0:1775764853.585030:0:22469:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac1473b58 00010000:00010000:2.0:1775764853.585034:0:22469:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764853.585043:0:22469:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764853.585058:0:22469:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:2.0:1775764853.585065:0:22469:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d11a.sanity-pfl, flags=24 00000080:00200000:2.0:1775764853.585844:0:22469:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88), inode mode 41ed mask 1 00000080:00200000:2.0:1775764853.585869:0:22469:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=f11a.sanity-pfl, dir=[0x200000401:0xedf:0x0](ffff8f9952eedb88), flags=8 00000080:00200000:2.0:1775764853.585874:0:22469:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=f11a.sanity-pfl, dir=[0x200000401:0xedf:0x0](ffff8f9952eedb88), intent=getattr 00000002:00010000:2.0:1775764853.585888:0:22469:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f11a.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000401:0xedf:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764853.585941:0:22469:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c93d lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 22469 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.585950:0:22469:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c93d lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 22469 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.585960:0:22469:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c93d lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 22469 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764853.585989:0:22469:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22469:1862022967045120:192.168.202.146@tcp:101:lfs.0 00000100:00100000:2.0:1775764853.586133:0:22469:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952af3d80 going to sleep for 11 seconds 00000100:00100000:2.0:1775764853.608962:0:22469:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22469:1862022967045120:192.168.202.146@tcp:101:lfs.0 00010000:00010000:2.0:1775764853.608984:0:22469:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c93d lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 22469 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.608997:0:22469:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c93d lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 22469 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.609005:0:22469:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c93d lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 22469 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.609012:0:22469:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c93d lrc: 3/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 22469 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.609021:0:22469:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c93d lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 22469 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.609027:0:22469:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99706ea600 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:2.0:1775764853.609032:0:22469:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c93d lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 22469 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.609039:0:22469:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c93d lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 22469 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.609047:0:22469:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c93d lrc: 5/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 22469 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.609054:0:22469:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1775764853.609061:0:22469:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c93d lrc: 3/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 22469 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.609067:0:22469:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c93d lrc: 0/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 22469 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764853.609085:0:22469:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=7, status=0 req@ffff8f99703d1c00 x1862022967045120/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/584 e 0 to 0 dl 1775764869 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764853.609101:0:22469:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0xedf:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1775764853.609125:0:22469:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/3807 (0 0) 00000080:00200000:2.0:1775764853.609200:0:22469:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:2.0:1775764853.609214:0:22469:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764853.609223:0:22469:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764853.609233:0:22469:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764853.609240:0:22469:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764853.609245:0:22469:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:2.0:1775764853.609248:0:22469:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:2.0:1775764853.609253:0:22469:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac1473bf8 00010000:00010000:2.0:1775764853.609256:0:22469:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764853.609265:0:22469:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1775764853.609280:0:22469:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:2.0:1775764853.609286:0:22469:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d11a.sanity-pfl, flags=25 00000080:00200000:2.0:1775764853.609289:0:22469:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88), inode mode 41ed mask 1 00000080:00200000:2.0:1775764853.609298:0:22469:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88), inode mode 41ed mask 3 00000080:00200000:2.0:1775764853.609301:0:22469:0:(namei.c:1399:ll_atomic_open()) VFS Op:name=f11a.sanity-pfl, dir=[0x200000401:0xedf:0x0](ffff8f9952eedb88), file ffff8f99435daf00, open_flags a141, mode 81b6 opened 0 00000080:00200000:2.0:1775764853.609307:0:22469:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=f11a.sanity-pfl, dir=[0x200000401:0xedf:0x0](ffff8f9952eedb88), intent=open|creat 00000002:00010000:2.0:1775764853.609322:0:22469:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f11a.sanity-pfl,[0x200000401:0xee0:0x0]) in obj [0x200000401:0xedf:0x0], intent: open|creat flags 0120502 00010000:00010000:2.0:1775764853.610548:0:22469:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c944 lrc: 3/0,1 mode: --/CW res: [0x200000401:0xedf:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 22469 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.610561:0:22469:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c944 lrc: 3/0,1 mode: --/CW res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 22469 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764853.610573:0:22469:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764853.610582:0:22469:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f99703d1c00 x1862022967045376/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 528/66536 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764853.610591:0:22469:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c944 lrc: 3/0,1 mode: --/CW res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 22469 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764853.610613:0:22469:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22469:1862022967045376:192.168.202.146@tcp:101:lfs.0 00000100:00100000:2.0:1775764853.611026:0:22469:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952af3d80 going to sleep for 11 seconds 00000100:00100000:2.0:1775764853.618487:0:22469:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22469:1862022967045376:192.168.202.146@tcp:101:lfs.0 00010000:00010000:2.0:1775764853.618511:0:22469:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c944 lrc: 4/0,1 mode: --/CW res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc1a9 expref: -99 pid: 22469 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.618529:0:22469:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c944 lrc: 4/0,1 mode: --/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc1a9 expref: -99 pid: 22469 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.618538:0:22469:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c944 lrc: 4/0,1 mode: --/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc1a9 expref: -99 pid: 22469 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.618551:0:22469:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c944 lrc: 4/0,1 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc1a9 expref: -99 pid: 22469 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.618561:0:22469:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c944 lrc: 4/0,1 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc1a9 expref: -99 pid: 22469 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.618573:0:22469:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c944 lrc: 5/1,1 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc1a9 expref: -99 pid: 22469 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.618582:0:22469:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c944 lrc: 5/1,1 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc1a9 expref: -99 pid: 22469 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.618588:0:22469:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c944 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc1a9 expref: -99 pid: 22469 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764853.618600:0:22469:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=3 disposition=8000037, status=0 req@ffff8f99703d1c00 x1862022967045376/t8589934613(8589934613) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 528/608 e 0 to 0 dl 1775764869 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764853.618613:0:22469:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f99703d1c00 x1862022967045376/t8589934613(8589934613) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 528/608 e 0 to 0 dl 1775764869 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764853.618625:0:22469:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c944 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc1a9 expref: -99 pid: 22469 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.618635:0:22469:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/3808 (0 0) 00000080:00200000:2.0:1775764853.618641:0:22469:0:(namei.c:943:ll_intent_lock()) intent lock 3 on i1 [0x200000401:0xedf:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764853.618654:0:22469:0:(namei.c:1610:ll_create_it()) VFS Op:name=f11a.sanity-pfl, dir=[0x200000401:0xedf:0x0](ffff8f9952eedb88), intent=open|creat 00000080:00200000:2.0:1775764853.618684:0:22469:0:(llite_lib.c:3177:ll_read_inode2()) VFS Op:inode=[0x200000401:0xee0:0x0](ffff8f9952eb7448) 00000080:00200000:2.0:1775764853.618691:0:22469:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0xee0:0x0], updating i_size 0 00000080:00200000:2.0:1775764853.618831:0:22469:0:(file.c:6766:ll_layout_conf()) [0x200000401:0xee0:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:2.0:1775764853.618842:0:22469:0:(namei.c:1576:ll_create_node()) setting l_ast_data to inode [0x200000401:0xee0:0x0](ffff8f9952eb7448) 00000080:00010000:2.0:1775764853.618845:0:22469:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xee0:0x0](ffff8f9952eb7448) for lock 0x36f9bdc9e421c944 00000080:00010000:2.0:1775764853.618863:0:22469:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xee0:0x0](ffff8f9952eb7448) 00000080:00200008:2.0:1775764853.618874:0:22469:0:(file.c:1057:ll_file_open()) START file f11a.sanity-pfl:[0x200000401:0xee0:0x0](ffff8f9952eb7448), flags 120501 00000002:00100000:2.0:1775764853.618908:0:22469:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f99703d1c00 x1862022967045376/t8589934613(8589934613) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 528/608 e 0 to 0 dl 1775764869 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1775764853.618944:0:22469:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c944 from it ffff8f9970a690c0 00010000:00010000:2.0:1775764853.618949:0:22469:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c944 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fc1a9 expref: -99 pid: 22469 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.618958:0:22469:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c944 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fc1a9 expref: -99 pid: 22469 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764853.618977:0:22469:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xee0:0x0](ffff8f9952eb7448), inode mode 81a4 mask 40 00000080:00200000:2.0:1775764853.619023:0:22469:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xee0:0x0](ffff8f9952eb7448), inode mode 81a4 mask 2 00000080:00200000:2.0:1775764853.619033:0:22469:0:(xattr.c:410:ll_xattr_set()) VFS Op:inode=[0x200000401:0xee0:0x0](ffff8f9952eb7448), xattr lov 00000002:00010000:2.0:1775764853.619048:0:22469:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xee0:0x0]) in obj [0x200000401:0xedf:0x0], intent: open flags 040000002 00010000:00010000:2.0:1775764853.619095:0:22469:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c94b lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 22469 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.619103:0:22469:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c94b lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 22469 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764853.619118:0:22469:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764853.619124:0:22469:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f99703d2300 x1862022967045632/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 952/132072 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764853.619133:0:22469:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c94b lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 22469 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764853.619152:0:22469:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22469:1862022967045632:192.168.202.146@tcp:101:lfs.0 00000100:00100000:2.0:1775764853.619323:0:22469:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952af3d80 going to sleep for 11 seconds 00000100:00100000:0.0:1775764853.623929:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764853.624468:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978417152 00000100:00100000:1.0:1775764853.624477:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978417152 00000100:00100000:1.0:1775764853.624555:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24226 00000100:00100000:1.0:1775764853.624565:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970a76d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+5:17506:x1862022978417152:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764853.624582:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c944 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x2aac6a50e5fc1a9 expref: -99 pid: 22469 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764853.624682:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764853.624689:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970a76d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+5:17506:x1862022978417152:12345-192.168.202.146@tcp:104: Request processed in 126us (772us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764853.624697:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24226 00010000:00010000:2.0:1775764853.624766:0:19800:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764853.624773:0:19800:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c944 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x2aac6a50e5fc1a9 expref: -99 pid: 22469 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764853.624782:0:19800:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99706eb800 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:2.0:1775764853.624788:0:19800:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c944 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x2aac6a50e5fc1a9 expref: -99 pid: 22469 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764853.624797:0:19800:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c944 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x2aac6a50e5fc1a9 expref: -99 pid: 22469 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764853.624806:0:19800:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c944 lrc: 3/0,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x2aac6a50e5fc1a9 expref: -99 pid: 22469 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764853.624872:0:19800:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0xee0:0x0]: losing layout lock 00010000:00010000:2.0:1775764853.624937:0:19800:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c944 lrc: 2/0,0 mode: --/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x2aac6a50e5fc1a9 expref: -99 pid: 22469 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764853.624944:0:19800:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764853.624974:0:19800:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c944 lrc: 1/0,0 mode: --/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x2aac6a50e5fc1a9 expref: -99 pid: 22469 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764853.624982:0:19800:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c944 lrc: 0/0,0 mode: --/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x2aac6a50e5fc1a9 expref: -99 pid: 22469 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764853.625000:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764853.625001:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764853.625003:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764853.625011:0:2381:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1775764853.625027:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967045760:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764853.627912:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967045760:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764853.629924:0:22469:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22469:1862022967045632:192.168.202.146@tcp:101:lfs.0 00010000:00010000:2.0:1775764853.629947:0:22469:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c94b lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 22469 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.629960:0:22469:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c94b lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 22469 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.629977:0:22469:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c94b lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 22469 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.629985:0:22469:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c94b lrc: 3/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 22469 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.629992:0:22469:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c94b lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 22469 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.629997:0:22469:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99706ea600 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:2.0:1775764853.630002:0:22469:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c94b lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 22469 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.630008:0:22469:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c94b lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 22469 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.630016:0:22469:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c94b lrc: 5/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 22469 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.630023:0:22469:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1775764853.630030:0:22469:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c94b lrc: 3/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 22469 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.630036:0:22469:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c94b lrc: 0/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 22469 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764853.630052:0:22469:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=800002b, status=0 req@ffff8f99703d2300 x1862022967045632/t8589934614(8589934614) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 952/1160 e 0 to 0 dl 1775764869 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764853.630065:0:22469:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f99703d2300 x1862022967045632/t8589934614(8589934614) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 952/1160 e 0 to 0 dl 1775764869 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764853.630089:0:22469:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0xedf:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764853.630176:0:22469:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xee0:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764853.630185:0:22469:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xee0:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764853.630269:0:22469:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0xee0:0x0], updating i_size 0 00000002:00100000:2.0:1775764853.630287:0:22469:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f99703d2300 x1862022967045632/t8589934614(8589934614) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1775764869 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764853.630323:0:22469:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f99703d2300 x1862022967045632/t8589934614(8589934614) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1775764869 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764853.630333:0:22469:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764853.630681:0:22469:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f99703d3480 x1862022967046016/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/840 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764853.630705:0:22469:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22469:1862022967046016:192.168.202.146@tcp:35:lfs.0 00000100:00100000:2.0:1775764853.631511:0:22469:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952af3d80 going to sleep for 11 seconds 00000100:00100000:2.0:1775764853.635484:0:22469:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22469:1862022967046016:192.168.202.146@tcp:35:lfs.0 00000002:00100000:2.0:1775764853.635524:0:22469:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f99703d2300 x1862022967045632/t8589934614(8589934614) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1775764869 ref 2 fl Complete:RQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200008:2.0:1775764853.635588:0:22469:0:(file.c:447:ll_file_release()) START file f11a.sanity-pfl:[0x200000401:0xee0:0x0](ffff8f9952eb7448), flags 100001 00000002:00100000:2.0:1775764853.635865:0:22469:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f99703d1c00 x1862022967045376/t8589934613(8589934613) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 528/608 e 0 to 0 dl 1775764869 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764853.635876:0:22469:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764853.635881:0:22469:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f99703d1180 x1862022967046272/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/840 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764853.635903:0:22469:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22469:1862022967046272:192.168.202.146@tcp:35:lfs.0 00000100:00100000:2.0:1775764853.636044:0:22469:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952af3d80 going to sleep for 11 seconds 00000100:00100000:2.0:1775764853.638340:0:22469:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22469:1862022967046272:192.168.202.146@tcp:35:lfs.0 00000002:00100000:2.0:1775764853.638374:0:22469:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f99703d1c00 x1862022967045376/t8589934613(8589934613) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 528/608 e 0 to 0 dl 1775764869 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1775764853.657409:0:22472:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:0.0:1775764853.657437:0:22472:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764853.657455:0:22472:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764853.657471:0:22472:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:0.0:1775764853.657478:0:22472:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764853.657487:0:22472:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:0.0:1775764853.657490:0:22472:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:0.0:1775764853.657497:0:22472:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac1e03bf8 00010000:00010000:0.0:1775764853.657500:0:22472:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1775764853.657509:0:22472:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:0.0:1775764853.657524:0:22472:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:0.0:1775764853.657531:0:22472:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d11a.sanity-pfl, flags=27 00000080:00200000:0.0:1775764853.657537:0:22472:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764853.657549:0:22472:0:(namei.c:1399:ll_atomic_open()) VFS Op:name=f11a.sanity-pfl, dir=[0x200000401:0xedf:0x0](ffff8f9952eedb88), file ffff8f9942fd8900, open_flags 18800, mode 0 opened 0 00000080:00200000:0.0:1775764853.657557:0:22472:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=f11a.sanity-pfl, dir=[0x200000401:0xedf:0x0](ffff8f9952eedb88), intent=open 00000002:00010000:0.0:1775764853.657573:0:22472:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f11a.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000401:0xedf:0x0], intent: open flags 0304001 00010000:00010000:0.0:1775764853.658028:0:22472:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c952 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 22472 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.658080:0:22472:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c952 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 22472 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1775764853.658092:0:22472:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1775764853.658104:0:22472:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970a77100 x1862022967046528/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 872/132072 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1775764853.658114:0:22472:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c952 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 22472 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764853.658137:0:22472:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22472:1862022967046528:192.168.202.146@tcp:101:lfs.0 00000100:00100000:0.0:1775764853.658406:0:22472:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102cc0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764853.661625:0:22472:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22472:1862022967046528:192.168.202.146@tcp:101:lfs.0 00010000:00010000:2.0:1775764853.661652:0:22472:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c952 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 22472 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.661665:0:22472:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c952 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 22472 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.661672:0:22472:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c952 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 22472 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.661678:0:22472:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c952 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 22472 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.661686:0:22472:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c952 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 22472 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.661693:0:22472:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99821cca00 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:2.0:1775764853.661697:0:22472:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c952 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 22472 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.661704:0:22472:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c952 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 22472 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.661710:0:22472:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c952 lrc: 5/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 22472 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.662043:0:22472:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1775764853.662053:0:22472:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c952 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 22472 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.662061:0:22472:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cca00/0x36f9bdc9e421c952 lrc: 0/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 22472 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764853.662083:0:22472:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff8f9970a77100 x1862022967046528/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 872/608 e 0 to 0 dl 1775764869 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764853.662097:0:22472:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0xedf:0x0] suppgids 0 -1: rc -20 00000080:00200000:2.0:1775764853.662242:0:22472:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:2.0:1775764853.662261:0:22472:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764853.662275:0:22472:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764853.662286:0:22472:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1775764853.662292:0:22472:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764853.662297:0:22472:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:2.0:1775764853.662301:0:22472:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:2.0:1775764853.662305:0:22472:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac1e03bf8 00010000:00010000:2.0:1775764853.662309:0:22472:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764853.662316:0:22472:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:2.0:1775764853.662329:0:22472:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:2.0:1775764853.662336:0:22472:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d11a.sanity-pfl, flags=267 00000080:00200000:2.0:1775764853.662343:0:22472:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88), inode mode 41ed mask 44 00000080:00200008:2.0:1775764853.662352:0:22472:0:(file.c:1057:ll_file_open()) START file d11a.sanity-pfl:[0x200000401:0xedf:0x0](ffff8f9952eedb88), flags 304000 00000002:00010000:2.0:1775764853.662366:0:22472:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xedf:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100240304001 00010000:00010000:2.0:1775764853.662413:0:22472:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c959 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 22472 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.662420:0:22472:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c959 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 22472 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764853.662431:0:22472:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764853.662439:0:22472:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f99703d1f80 x1862022967046784/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/132072 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764853.662448:0:22472:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c959 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 22472 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764853.662468:0:22472:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1f80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22472:1862022967046784:192.168.202.146@tcp:101:lfs.0 00000100:00100000:2.0:1775764853.665631:0:22472:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952af3d80 going to sleep for 11 seconds 00000100:00100000:2.0:1775764853.668588:0:22472:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1f80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22472:1862022967046784:192.168.202.146@tcp:101:lfs.0 00010000:00010000:2.0:1775764853.668626:0:22472:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c959 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 22472 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.668640:0:22472:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c959 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 22472 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.668646:0:22472:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c959 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 22472 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.668653:0:22472:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c959 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 22472 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.668661:0:22472:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c959 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 22472 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.668667:0:22472:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99706ea600 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:2.0:1775764853.668672:0:22472:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c959 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 22472 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.668678:0:22472:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c959 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 22472 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.668689:0:22472:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c959 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 22472 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.668695:0:22472:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1775764853.668701:0:22472:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c959 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 22472 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.668707:0:22472:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c959 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 22472 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764853.668762:0:22472:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8f99703d1f80 x1862022967046784/t8589934617(8589934617) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/608 e 0 to 0 dl 1775764869 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764853.668777:0:22472:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f99703d1f80 x1862022967046784/t8589934617(8589934617) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/608 e 0 to 0 dl 1775764869 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764853.668790:0:22472:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764853.668810:0:22472:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0xedf:0x0], updating i_size 512 00000002:00100000:2.0:1775764853.668826:0:22472:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f99703d1f80 x1862022967046784/t8589934617(8589934617) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/608 e 0 to 0 dl 1775764869 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:2.0:1775764853.668934:0:22472:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88) cmd=c0086915 arg=131c3d0 00000100:00100000:2.0:1775764853.668993:0:22472:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22472:1862022967046912:192.168.202.146@tcp:34:lfs.0 00000100:00100000:2.0:1775764853.669080:0:22472:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952af3d80 going to sleep for 11 seconds 00000100:00100000:2.0:1775764853.672672:0:22472:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22472:1862022967046912:192.168.202.146@tcp:34:lfs.0 00000080:00200008:2.0:1775764853.672940:0:22472:0:(file.c:447:ll_file_release()) START file d11a.sanity-pfl:[0x200000401:0xedf:0x0](ffff8f9952eedb88), flags 304000 00000002:00100000:2.0:1775764853.673000:0:22472:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f99703d1f80 x1862022967046784/t8589934617(8589934617) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/608 e 0 to 0 dl 1775764869 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764853.673018:0:22472:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764853.673023:0:22472:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f99703d2a00 x1862022967047168/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/840 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764853.673044:0:22472:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22472:1862022967047168:192.168.202.146@tcp:35:lfs.0 00000100:00100000:2.0:1775764853.673128:0:22472:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952af3d80 going to sleep for 11 seconds 00000100:00100000:2.0:1775764853.676583:0:22472:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22472:1862022967047168:192.168.202.146@tcp:35:lfs.0 00000002:00100000:2.0:1775764853.676632:0:22472:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f99703d1f80 x1862022967046784/t8589934617(8589934617) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/608 e 0 to 0 dl 1775764869 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764853.694330:0:22475:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764853.694486:0:22475:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764853.694501:0:22475:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764853.694514:0:22475:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:1.0:1775764853.694519:0:22475:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764853.694527:0:22475:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764853.694530:0:22475:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:1.0:1775764853.694573:0:22475:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac1d83bf8 00010000:00010000:1.0:1775764853.694576:0:22475:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764853.694582:0:22475:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1775764853.694594:0:22475:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764853.694600:0:22475:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d11a.sanity-pfl, flags=27 00000080:00200000:1.0:1775764853.694604:0:22475:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764853.694613:0:22475:0:(namei.c:1399:ll_atomic_open()) VFS Op:name=f11a.sanity-pfl, dir=[0x200000401:0xedf:0x0](ffff8f9952eedb88), file ffff8f9945160780, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1775764853.694618:0:22475:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=f11a.sanity-pfl, dir=[0x200000401:0xedf:0x0](ffff8f9952eedb88), intent=open 00000002:00010000:1.0:1775764853.694632:0:22475:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f11a.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000401:0xedf:0x0], intent: open flags 0304001 00010000:00010000:1.0:1775764853.694700:0:22475:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c960 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 22475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764853.694708:0:22475:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c960 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 22475 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764853.694719:0:22475:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764853.695186:0:22475:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e5c00 x1862022967047424/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 872/132072 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764853.695200:0:22475:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c960 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 22475 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764853.695228:0:22475:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5c00 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22475:1862022967047424:192.168.202.146@tcp:101:lfs.0 00000100:00100000:1.0:1775764853.695941:0:22475:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560600 going to sleep for 11 seconds 00000100:00100000:1.0:1775764853.699564:0:22475:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5c00 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22475:1862022967047424:192.168.202.146@tcp:101:lfs.0 00010000:00010000:1.0:1775764853.699627:0:22475:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c960 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 22475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764853.699640:0:22475:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c960 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 22475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764853.699647:0:22475:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c960 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 22475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764853.699654:0:22475:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c960 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 22475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764853.699663:0:22475:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c960 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 22475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764853.699669:0:22475:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970641600 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:1.0:1775764853.699674:0:22475:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c960 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 22475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764853.699681:0:22475:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c960 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 22475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764853.699688:0:22475:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c960 lrc: 5/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 22475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764853.699694:0:22475:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1775764853.699700:0:22475:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c960 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 22475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764853.699707:0:22475:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641600/0x36f9bdc9e421c960 lrc: 0/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 22475 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764853.700144:0:22475:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff8f994f2e5c00 x1862022967047424/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 872/608 e 0 to 0 dl 1775764869 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764853.700158:0:22475:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0xedf:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1775764853.700334:0:22475:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764853.700354:0:22475:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764853.700371:0:22475:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764853.700384:0:22475:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1775764853.700392:0:22475:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764853.700398:0:22475:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764853.700401:0:22475:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:1.0:1775764853.700407:0:22475:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac1d83bf8 00010000:00010000:1.0:1775764853.700411:0:22475:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764853.700419:0:22475:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1775764853.700433:0:22475:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764853.700439:0:22475:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d11a.sanity-pfl, flags=267 00000080:00200000:1.0:1775764853.700444:0:22475:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88), inode mode 41ed mask 44 00000080:00200008:1.0:1775764853.700451:0:22475:0:(file.c:1057:ll_file_open()) START file d11a.sanity-pfl:[0x200000401:0xedf:0x0](ffff8f9952eedb88), flags 304000 00000002:00010000:1.0:1775764853.700464:0:22475:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xedf:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0104240304001 00010000:00010000:1.0:1775764853.700512:0:22475:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640600/0x36f9bdc9e421c967 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 22475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764853.700517:0:22475:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640600/0x36f9bdc9e421c967 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 22475 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764853.700525:0:22475:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764853.700529:0:22475:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e5c00 x1862022967047680/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/132072 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764853.700536:0:22475:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640600/0x36f9bdc9e421c967 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 22475 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764853.700552:0:22475:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5c00 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22475:1862022967047680:192.168.202.146@tcp:101:lfs.0 00000100:00100000:1.0:1775764853.700701:0:22475:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560600 going to sleep for 11 seconds 00000100:00100000:1.0:1775764853.707255:0:22475:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5c00 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22475:1862022967047680:192.168.202.146@tcp:101:lfs.0 00010000:00010000:1.0:1775764853.707291:0:22475:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640600/0x36f9bdc9e421c967 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc220 expref: -99 pid: 22475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764853.707303:0:22475:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640600/0x36f9bdc9e421c967 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc220 expref: -99 pid: 22475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764853.707315:0:22475:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640600/0x36f9bdc9e421c967 lrc: 4/1,0 mode: CR/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc220 expref: -99 pid: 22475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764853.707325:0:22475:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640600/0x36f9bdc9e421c967 lrc: 4/1,0 mode: CR/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc220 expref: -99 pid: 22475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764853.707340:0:22475:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=200002b, status=0 req@ffff8f994f2e5c00 x1862022967047680/t8589934619(8589934619) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/608 e 0 to 0 dl 1775764869 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764853.707396:0:22475:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f994f2e5c00 x1862022967047680/t8589934619(8589934619) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/608 e 0 to 0 dl 1775764869 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764853.707407:0:22475:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640600/0x36f9bdc9e421c967 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc220 expref: -99 pid: 22475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764853.707417:0:22475:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 16 res 8589935617/3807 (0 0) 00000080:00200000:1.0:1775764853.707424:0:22475:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1775764853.707446:0:22475:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0xedf:0x0], updating i_size 512 00000080:00010000:1.0:1775764853.707451:0:22475:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xedf:0x0](ffff8f9952eedb88) for lock 0x36f9bdc9e421c967 00000080:00010000:1.0:1775764853.707458:0:22475:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c967 from it ffffafbac1d83bd0 00010000:00010000:1.0:1775764853.707462:0:22475:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640600/0x36f9bdc9e421c967 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc220 expref: -99 pid: 22475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764853.707472:0:22475:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640600/0x36f9bdc9e421c967 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc220 expref: -99 pid: 22475 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000002:00100000:1.0:1775764853.707500:0:22475:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f994f2e5c00 x1862022967047680/t8589934619(8589934619) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/608 e 0 to 0 dl 1775764869 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:1.0:1775764853.707606:0:22475:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88) cmd=c0086915 arg=ae23d0 00000100:00100000:1.0:1775764853.707976:0:22475:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22475:1862022967047808:192.168.202.146@tcp:34:lfs.0 00000100:00100000:1.0:1775764853.708136:0:22475:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560600 going to sleep for 11 seconds 00000100:00100000:3.0:1775764853.712065:0:22475:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22475:1862022967047808:192.168.202.146@tcp:34:lfs.0 00000080:00200008:3.0:1775764853.712309:0:22475:0:(file.c:447:ll_file_release()) START file d11a.sanity-pfl:[0x200000401:0xedf:0x0](ffff8f9952eedb88), flags 304000 00010000:00010000:3.0:1775764853.712340:0:22475:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640600/0x36f9bdc9e421c967 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc220 expref: -99 pid: 22475 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764853.730503:0:22478:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:2.0:1775764853.730528:0:22478:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764853.730545:0:22478:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:2.0:1775764853.730557:0:22478:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:2.0:1775764853.730563:0:22478:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764853.730569:0:22478:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:2.0:1775764853.730571:0:22478:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:2.0:1775764853.730577:0:22478:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac1d7bbf8 00010000:00010000:2.0:1775764853.730579:0:22478:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:2.0:1775764853.730585:0:22478:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:2.0:1775764853.730600:0:22478:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:2.0:1775764853.730606:0:22478:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d11a.sanity-pfl, flags=27 00000080:00200000:2.0:1775764853.730610:0:22478:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88), inode mode 41ed mask 1 00000080:00200000:2.0:1775764853.730620:0:22478:0:(namei.c:1399:ll_atomic_open()) VFS Op:name=f11a.sanity-pfl, dir=[0x200000401:0xedf:0x0](ffff8f9952eedb88), file ffff8f9942ed6a80, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1775764853.730627:0:22478:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=f11a.sanity-pfl, dir=[0x200000401:0xedf:0x0](ffff8f9952eedb88), intent=open 00000002:00010000:2.0:1775764853.730641:0:22478:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f11a.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000401:0xedf:0x0], intent: open flags 0304001 00010000:00010000:2.0:1775764853.730699:0:22478:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c96e lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 22478 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.730707:0:22478:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c96e lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 22478 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764853.730718:0:22478:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764853.730730:0:22478:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f99703d2a00 x1862022967048064/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 872/132072 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764853.738346:0:22478:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c96e lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 22478 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764853.738439:0:22478:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22478:1862022967048064:192.168.202.146@tcp:101:lfs.0 00000100:00100000:2.0:1775764853.738614:0:22478:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952af3d80 going to sleep for 11 seconds 00000100:00100000:2.0:1775764853.747278:0:22478:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22478:1862022967048064:192.168.202.146@tcp:101:lfs.0 00010000:00010000:2.0:1775764853.747301:0:22478:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c96e lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 22478 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.747314:0:22478:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c96e lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 22478 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.747326:0:22478:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c96e lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 22478 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.747333:0:22478:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c96e lrc: 3/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 22478 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.747340:0:22478:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c96e lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 22478 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.747346:0:22478:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99706ea600 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:2.0:1775764853.747351:0:22478:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c96e lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 22478 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.747357:0:22478:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c96e lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 22478 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.747363:0:22478:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c96e lrc: 5/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 22478 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.747369:0:22478:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1775764853.747375:0:22478:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c96e lrc: 3/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 22478 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.747381:0:22478:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c96e lrc: 0/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 22478 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764853.747398:0:22478:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff8f99703d2a00 x1862022967048064/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 872/608 e 0 to 0 dl 1775764869 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764853.747413:0:22478:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0xedf:0x0] suppgids 0 -1: rc -20 00000080:00200000:2.0:1775764853.747544:0:22478:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:2.0:1775764853.747562:0:22478:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764853.747576:0:22478:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:2.0:1775764853.747587:0:22478:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:2.0:1775764853.747594:0:22478:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764853.747599:0:22478:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:2.0:1775764853.747602:0:22478:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:2.0:1775764853.747607:0:22478:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac1d7bbf8 00010000:00010000:2.0:1775764853.747609:0:22478:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:2.0:1775764853.747618:0:22478:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:2.0:1775764853.747631:0:22478:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:2.0:1775764853.747638:0:22478:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d11a.sanity-pfl, flags=267 00000080:00200000:2.0:1775764853.747644:0:22478:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88), inode mode 41ed mask 44 00000080:00200008:2.0:1775764853.747652:0:22478:0:(file.c:1057:ll_file_open()) START file d11a.sanity-pfl:[0x200000401:0xedf:0x0](ffff8f9952eedb88), flags 304000 00000080:00200080:2.0:1775764853.747686:0:22478:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88) cmd=c0086915 arg=23943d0 00000100:00100000:2.0:1775764853.748038:0:22478:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22478:1862022967048192:192.168.202.146@tcp:34:lfs.0 00000100:00100000:0.0:1775764853.748172:0:22478:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952af3d80 going to sleep for 11 seconds 00000100:00100000:0.0:1775764853.752395:0:22478:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22478:1862022967048192:192.168.202.146@tcp:34:lfs.0 00000080:00200008:1.0:1775764853.755329:0:22478:0:(file.c:447:ll_file_release()) START file d11a.sanity-pfl:[0x200000401:0xedf:0x0](ffff8f9952eedb88), flags 304000 00010000:00010000:1.0:1775764853.755370:0:22478:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640600/0x36f9bdc9e421c967 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc220 expref: -99 pid: 22475 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764853.772872:0:22481:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764853.772940:0:22481:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764853.772959:0:22481:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764853.772974:0:22481:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:1.0:1775764853.772982:0:22481:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764853.772992:0:22481:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764853.772996:0:22481:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:1.0:1775764853.773003:0:22481:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac1df3bf8 00010000:00010000:1.0:1775764853.773007:0:22481:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764853.773016:0:22481:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00000080:00200000:1.0:1775764853.773031:0:22481:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764853.773039:0:22481:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d11a.sanity-pfl, flags=27 00000080:00200000:1.0:1775764853.773045:0:22481:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764853.773056:0:22481:0:(namei.c:1399:ll_atomic_open()) VFS Op:name=f11a.sanity-pfl, dir=[0x200000401:0xedf:0x0](ffff8f9952eedb88), file ffff8f9960492600, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1775764853.773062:0:22481:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=f11a.sanity-pfl, dir=[0x200000401:0xedf:0x0](ffff8f9952eedb88), intent=open 00000002:00010000:1.0:1775764853.773075:0:22481:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f11a.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000401:0xedf:0x0], intent: open flags 0304001 00010000:00010000:1.0:1775764853.773125:0:22481:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c975 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 22481 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764853.773133:0:22481:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c975 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 22481 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764853.773163:0:22481:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764853.773716:0:22481:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e5880 x1862022967048448/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 872/132072 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764853.773879:0:22481:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c975 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 22481 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764853.773906:0:22481:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5880 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22481:1862022967048448:192.168.202.146@tcp:101:lfs.0 00000100:00100000:1.0:1775764853.775017:0:22481:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560b40 going to sleep for 11 seconds 00000100:00100000:2.0:1775764853.785307:0:22481:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5880 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22481:1862022967048448:192.168.202.146@tcp:101:lfs.0 00010000:00010000:2.0:1775764853.785331:0:22481:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c975 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 22481 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.785343:0:22481:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c975 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 22481 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.785351:0:22481:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c975 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 22481 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.785357:0:22481:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c975 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 22481 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.785365:0:22481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c975 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 22481 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.785371:0:22481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970640000 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:2.0:1775764853.785375:0:22481:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c975 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 22481 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.785382:0:22481:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c975 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 22481 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.785389:0:22481:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c975 lrc: 5/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 22481 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.785396:0:22481:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1775764853.785402:0:22481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c975 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 22481 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.785408:0:22481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640000/0x36f9bdc9e421c975 lrc: 0/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 22481 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764853.785425:0:22481:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff8f994f2e5880 x1862022967048448/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 872/608 e 0 to 0 dl 1775764869 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764853.785436:0:22481:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0xedf:0x0] suppgids 0 -1: rc -20 00000080:00200000:2.0:1775764853.785560:0:22481:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:2.0:1775764853.785575:0:22481:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764853.785587:0:22481:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00010000:00010000:2.0:1775764853.785595:0:22481:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00000080:00200000:2.0:1775764853.785600:0:22481:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764853.785604:0:22481:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:2.0:1775764853.785607:0:22481:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:2.0:1775764853.785611:0:22481:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac1df3bf8 00010000:00010000:2.0:1775764853.785614:0:22481:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00010000:00010000:2.0:1775764853.785622:0:22481:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 0 00000080:00200000:2.0:1775764853.785635:0:22481:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:2.0:1775764853.785642:0:22481:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d11a.sanity-pfl, flags=267 00000080:00200000:2.0:1775764853.785648:0:22481:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88), inode mode 41ed mask 44 00000080:00200008:2.0:1775764853.785655:0:22481:0:(file.c:1057:ll_file_open()) START file d11a.sanity-pfl:[0x200000401:0xedf:0x0](ffff8f9952eedb88), flags 304000 00000080:00200080:2.0:1775764853.785690:0:22481:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88) cmd=c0086915 arg=14013d0 00000100:00100000:2.0:1775764853.785780:0:22481:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1f80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22481:1862022967048576:192.168.202.146@tcp:34:lfs.0 00000100:00100000:2.0:1775764853.785877:0:22481:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952af3180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764853.791013:0:22481:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1f80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22481:1862022967048576:192.168.202.146@tcp:34:lfs.0 00000080:00200008:2.0:1775764853.791273:0:22481:0:(file.c:447:ll_file_release()) START file d11a.sanity-pfl:[0x200000401:0xedf:0x0](ffff8f9952eedb88), flags 304000 00010000:00010000:2.0:1775764853.791306:0:22481:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640600/0x36f9bdc9e421c967 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc220 expref: -99 pid: 22475 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:0.0:1775764853.806992:0:22483:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:0.0:1775764853.807024:0:22483:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764853.807043:0:22483:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 0 00010000:00010000:0.0:1775764853.807058:0:22483:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 0 00000080:00200000:0.0:1775764853.807065:0:22483:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764853.807073:0:22483:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:0.0:1775764853.807076:0:22483:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:0.0:1775764853.807082:0:22483:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac1df3c08 00010000:00010000:0.0:1775764853.807086:0:22483:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 0 00010000:00010000:0.0:1775764853.807134:0:22483:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:0.0:1775764853.807150:0:22483:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:0.0:1775764853.807158:0:22483:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d11a.sanity-pfl, flags=25 00000080:00200000:0.0:1775764853.807164:0:22483:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764853.807174:0:22483:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=f11a.sanity-pfl, dir=[0x200000401:0xedf:0x0](ffff8f9952eedb88), flags=9 00000080:00200000:0.0:1775764853.807178:0:22483:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=f11a.sanity-pfl, dir=[0x200000401:0xedf:0x0](ffff8f9952eedb88), intent=getattr 00000002:00010000:0.0:1775764853.807190:0:22483:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f11a.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000401:0xedf:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764853.807238:0:22483:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421c97c lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 22483 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.807246:0:22483:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421c97c lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 22483 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.807256:0:22483:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421c97c lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 22483 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764853.807282:0:22483:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job truncate:3f83546c-01a9-4156-8a97-44c446245d3e:22483:1862022967048704:192.168.202.146@tcp:101:truncate.0 00000100:00100000:0.0:1775764853.807388:0:22483:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102300 going to sleep for 11 seconds 00000100:00100000:2.0:1775764853.813060:0:22483:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job truncate:3f83546c-01a9-4156-8a97-44c446245d3e:22483:1862022967048704:192.168.202.146@tcp:101:truncate.0 00010000:00010000:2.0:1775764853.813085:0:22483:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421c97c lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc2ac expref: -99 pid: 22483 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.813117:0:22483:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421c97c lrc: 4/1,0 mode: --/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc2ac expref: -99 pid: 22483 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.813126:0:22483:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421c97c lrc: 4/1,0 mode: --/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc2ac expref: -99 pid: 22483 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.813137:0:22483:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421c97c lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc2ac expref: -99 pid: 22483 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.813147:0:22483:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421c97c lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc2ac expref: -99 pid: 22483 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.813159:0:22483:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421c97c lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc2ac expref: -99 pid: 22483 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.813168:0:22483:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421c97c lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc2ac expref: -99 pid: 22483 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.813184:0:22483:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421c97c lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc2ac expref: -99 pid: 22483 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764853.813196:0:22483:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970a77100 x1862022967048704/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/1136 e 0 to 0 dl 1775764869 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'truncate.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764853.813209:0:22483:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421c97c lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc2ac expref: -99 pid: 22483 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764853.813222:0:22483:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421c97c lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc2ac expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.813233:0:22483:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/3808 (0 0) 00000080:00200000:2.0:1775764853.813239:0:22483:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0xedf:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764853.813326:0:22483:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0xee0:0x0](ffff8f9952eb7448): rc = 0 00000080:00200000:2.0:1775764853.813349:0:22483:0:(file.c:6766:ll_layout_conf()) [0x200000401:0xee0:0x0]: layout version change: 4294967294 -> 5 00000080:00010000:2.0:1775764853.813360:0:22483:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xee0:0x0](ffff8f9952eb7448) for lock 0x36f9bdc9e421c97c 00000080:00010000:2.0:1775764853.813371:0:22483:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xee0:0x0](ffff8f9952eb7448) 00000080:00010000:2.0:1775764853.813374:0:22483:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c97c from it ffffafbac1df3bd0 00010000:00010000:2.0:1775764853.813377:0:22483:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421c97c lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fc2ac expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.813386:0:22483:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421c97c lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fc2ac expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764853.813423:0:22483:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xee0:0x0](ffff8f9952eb7448), inode mode 81a4 mask 2 00000080:00200008:2.0:1775764853.813436:0:22483:0:(llite_lib.c:2274:ll_setattr_raw()) START file f11a.sanity-pfl:[0x200000401:0xee0:0x0](ffff8f9952eb7448) current size 0, valid attrs 8, mode ffff, uid -1734053588, gid -1, new size 1048576, atime 1775764853.000000000, mtime 1775764853.000000000, ctime 1775764853.000000000 00010000:00010000:2.0:1775764853.813460:0:22483:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421c97c lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5fc2ac expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764853.813484:0:22483:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0xee0:0x0]: losing layout lock 00010000:00010000:2.0:1775764853.813529:0:22483:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421c97c lrc: 1/0,0 mode: --/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fc2ac expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764853.813537:0:22483:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764853.813539:0:22483:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421c97c lrc: 0/0,0 mode: --/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fc2ac expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000020:00100000:2.0:1775764853.813563:0:22483:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1775764853.813570:0:22483:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f99703d1f80 x1862022967048960/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764853.813591:0:22483:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1f80 pname:cluuid:pid:xid:nid:opc:job truncate:3f83546c-01a9-4156-8a97-44c446245d3e:22483:1862022967048960:192.168.202.146@tcp:36:truncate.0 00000100:00100000:2.0:1775764853.813673:0:22483:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952af3180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764853.817394:0:22483:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1f80 pname:cluuid:pid:xid:nid:opc:job truncate:3f83546c-01a9-4156-8a97-44c446245d3e:22483:1862022967048960:192.168.202.146@tcp:36:truncate.0 00000080:00200000:2.0:1775764853.817471:0:22483:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xee0:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00010000:2.0:1775764853.817490:0:22483:0:(file.c:6988:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200000401:0xee0:0x0](ffff8f9952eb7448) 00000002:00010000:2.0:1775764853.817500:0:22483:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xee0:0x0]) in obj [0x200000401:0xee0:0x0], intent: layout flags 0 00010000:00010000:2.0:1775764853.817550:0:22483:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c983 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 22483 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.817560:0:22483:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c983 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 22483 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.817570:0:22483:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c983 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 22483 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764853.817588:0:22483:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0a80 pname:cluuid:pid:xid:nid:opc:job truncate:3f83546c-01a9-4156-8a97-44c446245d3e:22483:1862022967049088:192.168.202.146@tcp:101:truncate.0 00000100:00100000:2.0:1775764853.817665:0:22483:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952af3180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764853.824593:0:22483:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0a80 pname:cluuid:pid:xid:nid:opc:job truncate:3f83546c-01a9-4156-8a97-44c446245d3e:22483:1862022967049088:192.168.202.146@tcp:101:truncate.0 00010000:00010000:2.0:1775764853.824649:0:22483:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c983 lrc: 4/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc2ba expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.824666:0:22483:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c983 lrc: 4/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc2ba expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764853.824682:0:22483:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8f99703d0a80 x1862022967049088/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 376/896 e 0 to 0 dl 1775764869 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'truncate.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764853.824694:0:22483:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c983 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc2ba expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764853.824712:0:22483:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c983 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc2ba expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.825421:0:22483:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 16 res 8589935617/3808 (0 0) 00000080:00200000:2.0:1775764853.825430:0:22483:0:(namei.c:943:ll_intent_lock()) intent lock 1024 on i1 [0x200000401:0xee0:0x0] suppgids 0 0: rc 0 00000080:00010000:2.0:1775764853.825451:0:22483:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xee0:0x0](ffff8f9952eb7448) for lock 0x36f9bdc9e421c983 00000080:00010000:2.0:1775764853.825460:0:22483:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c983 from it ffffafbac1df3ad8 00010000:00010000:2.0:1775764853.825465:0:22483:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c983 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc2ba expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.825475:0:22483:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c983 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc2ba expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764853.825501:0:22483:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c983 lrc: 2/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc2ba expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764853.825511:0:22483:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c983 lrc: 2/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc2ba expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:2.0:1775764853.825519:0:22483:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200000401:0xee0:0x0](000000002c188e9c) being reconfigured ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c983 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc2ba expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764853.825583:0:22483:0:(file.c:6766:ll_layout_conf()) [0x200000401:0xee0:0x0]: layout version change: 4294967294 -> 5 00010000:00010000:2.0:1775764853.825603:0:22483:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c983 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fc2ba expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764853.825701:0:22483:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c983 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fc2ba expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764853.826576:0:22483:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xee0:0x0] ignore/verify layout 0/0, layout version 5 need write layout 1, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764853.826583:0:22483:0:(vvp_io.c:355:vvp_io_fini()) [0x200000401:0xee0:0x0] write layout, type 3 [0x0, 0x100001) 00000080:00010000:2.0:1775764853.826594:0:22483:0:(file.c:6988:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200000401:0xee0:0x0](ffff8f9952eb7448) 00000002:00010000:2.0:1775764853.826604:0:22483:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xee0:0x0]) in obj [0x200000401:0xee0:0x0], intent: layout flags 02 00010000:00010000:2.0:1775764853.826632:0:22483:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c983 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5fc2ba expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764853.826668:0:22483:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0xee0:0x0]: losing layout lock 00010000:00010000:2.0:1775764853.826724:0:22483:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c983 lrc: 1/0,0 mode: --/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fc2ba expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764853.826983:0:22483:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764853.826987:0:22483:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c983 lrc: 0/0,0 mode: --/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fc2ba expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764853.827024:0:22483:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(EX) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c98a lrc: 3/0,1 mode: --/EX res: [0x200000401:0xee0:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 22483 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.827036:0:22483:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c98a lrc: 3/0,1 mode: --/EX res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 22483 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764853.827049:0:22483:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764853.827060:0:22483:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f99703d0a80 x1862022967049344/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 376/896 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764853.827069:0:22483:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c98a lrc: 3/0,1 mode: --/EX res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 22483 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764853.827089:0:22483:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0a80 pname:cluuid:pid:xid:nid:opc:job truncate:3f83546c-01a9-4156-8a97-44c446245d3e:22483:1862022967049344:192.168.202.146@tcp:101:truncate.0 00000100:00100000:2.0:1775764853.827397:0:22483:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952af3180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764853.834458:0:22483:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0a80 pname:cluuid:pid:xid:nid:opc:job truncate:3f83546c-01a9-4156-8a97-44c446245d3e:22483:1862022967049344:192.168.202.146@tcp:101:truncate.0 00010000:00010000:2.0:1775764853.834484:0:22483:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode CR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c98a lrc: 4/0,1 mode: --/EX res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc2c8 expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.834497:0:22483:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c98a lrc: 4/0,1 mode: --/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc2c8 expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.834507:0:22483:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c98a lrc: 4/0,1 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc2c8 expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.834518:0:22483:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c98a lrc: 4/0,1 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc2c8 expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.834530:0:22483:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c98a lrc: 5/1,1 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc2c8 expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.834542:0:22483:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(EX) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c98a lrc: 5/1,1 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc2c8 expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.834549:0:22483:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c98a lrc: 4/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc2c8 expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764853.834560:0:22483:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8f99703d0a80 x1862022967049344/t8589934622(8589934622) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 376/896 e 0 to 0 dl 1775764869 ref 2 fl Complete:RQU/604/0 rc 0/0 job:'truncate.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764853.834579:0:22483:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c98a lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc2c8 expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764853.834590:0:22483:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c98a lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc2c8 expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.834603:0:22483:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 16 res 8589935617/3808 (0 0) 00000080:00200000:2.0:1775764853.835008:0:22483:0:(namei.c:943:ll_intent_lock()) intent lock 1024 on i1 [0x200000401:0xee0:0x0] suppgids 0 0: rc 0 00000080:00010000:2.0:1775764853.835019:0:22483:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xee0:0x0](ffff8f9952eb7448) for lock 0x36f9bdc9e421c98a 00000080:00010000:2.0:1775764853.835028:0:22483:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c98a from it ffffafbac1df3b40 00010000:00010000:2.0:1775764853.835031:0:22483:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c98a lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc2c8 expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.835040:0:22483:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c98a lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc2c8 expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764853.835061:0:22483:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xee0:0x0] ignore/verify layout 0/0, layout version 5 restore needed 0 00010000:00010000:0.0:1775764853.835168:0:22483:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c98a lrc: 2/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc2c8 expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764853.835443:0:22483:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c98a lrc: 2/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc2c8 expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:0.0:1775764853.835453:0:22483:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200000401:0xee0:0x0](000000002c188e9c) being reconfigured ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c98a lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc2c8 expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764853.835584:0:22483:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c98a lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc2c8 expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764853.835592:0:22483:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c98a lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc2c8 expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764853.836072:0:22483:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xee0:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764853.836085:0:22483:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1775764853.836117:0:22483:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:0.0:1775764853.836120:0:22483:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00020000:00200000:0.0:1775764853.836121:0:22483:0:(lov_io.c:860:lov_io_iter_init()) component[3] flags 0x0 00020000:00200000:0.0:1775764853.836122:0:22483:0:(lov_io.c:860:lov_io_iter_init()) component[4] flags 0x0 00000080:00200000:0.0:1775764853.836154:0:22483:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xee0:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1775764853.836431:0:22483:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c98a lrc: 2/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc2c8 expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764853.836444:0:22483:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c98a lrc: 2/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc2c8 expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:0.0:1775764853.836452:0:22483:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200000401:0xee0:0x0](000000002c188e9c) being reconfigured ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c98a lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc2c8 expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764853.836501:0:22483:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xee0:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764853.836506:0:22483:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1775764853.836539:0:22483:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:0.0:1775764853.836541:0:22483:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00020000:00200000:0.0:1775764853.836542:0:22483:0:(lov_io.c:860:lov_io_iter_init()) component[3] flags 0x0 00020000:00200000:0.0:1775764853.836543:0:22483:0:(lov_io.c:860:lov_io_iter_init()) component[4] flags 0x0 00000080:00200000:0.0:1775764853.836558:0:22483:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xee0:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764853.837106:0:22483:0:(file.c:6766:ll_layout_conf()) [0x200000401:0xee0:0x0]: layout version change: 4294967294 -> 6 00010000:00010000:0.0:1775764853.837117:0:22483:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c98a lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fc2c8 expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764853.837126:0:22483:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c98a lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fc2c8 expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00020000:00200000:0.0:1775764853.837152:0:22483:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:0.0:1775764853.837169:0:22483:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00020000:00200000:0.0:1775764853.837170:0:22483:0:(lov_io.c:860:lov_io_iter_init()) component[3] flags 0x0 00020000:00200000:0.0:1775764853.837171:0:22483:0:(lov_io.c:860:lov_io_iter_init()) component[4] flags 0x0 00000080:00200000:0.0:1775764853.837315:0:22483:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:0.0:1775764853.837442:0:22483:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764853.837448:0:22483:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f9960102db0, osc lock ffff8f996686ee40, flags 0x0 00000008:00010000:0.0:1775764853.837452:0:22483:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764853.837454:0:22483:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f9960102df0, osc lock ffff8f996686f560, flags 0x0 00010000:00010000:0.0:1775764853.837509:0:22483:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 3/0,1 mode: --/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.837517:0:22483:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 3/0,1 mode: --/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.837527:0:22483:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 3/0,1 mode: --/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764853.837555:0:22483:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74e00 pname:cluuid:pid:xid:nid:opc:job truncate:3f83546c-01a9-4156-8a97-44c446245d3e:22483:1862022967049472:192.168.202.146@tcp:101:truncate.0 00000100:00100000:0.0:1775764853.838008:0:22483:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102480 going to sleep for 11 seconds 00000100:00100000:0.0:1775764853.842587:0:22483:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74e00 pname:cluuid:pid:xid:nid:opc:job truncate:3f83546c-01a9-4156-8a97-44c446245d3e:22483:1862022967049472:192.168.202.146@tcp:101:truncate.0 00010000:00010000:0.0:1775764853.842707:0:22483:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 4/0,1 mode: --/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.843156:0:22483:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.843169:0:22483:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.843200:0:22483:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 5/0,2 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1775764853.843215:0:22483:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 4/0,2 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.843227:0:22483:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 5/0,2 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.843234:0:22483:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.843308:0:22483:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 3/0,1 mode: --/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.843317:0:22483:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 3/0,1 mode: --/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [1048576->2097151] (req 1048576->2097151) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.843328:0:22483:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 3/0,1 mode: --/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [1048576->2097151] (req 1048576->2097151) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764853.843378:0:22483:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74e00 pname:cluuid:pid:xid:nid:opc:job truncate:3f83546c-01a9-4156-8a97-44c446245d3e:22483:1862022967049600:192.168.202.146@tcp:101:truncate.0 00000100:00100000:0.0:1775764853.843462:0:22483:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102480 going to sleep for 11 seconds 00000100:00100000:0.0:1775764853.851292:0:22483:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74e00 pname:cluuid:pid:xid:nid:opc:job truncate:3f83546c-01a9-4156-8a97-44c446245d3e:22483:1862022967049600:192.168.202.146@tcp:101:truncate.0 00010000:00010000:0.0:1775764853.851319:0:22483:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 4/0,1 mode: --/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.851347:0:22483:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 4/0,1 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.851359:0:22483:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 4/0,1 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.851373:0:22483:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 5/0,2 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1775764853.851387:0:22483:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 4/0,2 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.851398:0:22483:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 5/0,2 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764853.851405:0:22483:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 4/0,1 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764853.851898:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967049728:192.168.202.146@tcp:10:truncate.0 00000100:00100000:0.0:1775764853.860293:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967049728:192.168.202.146@tcp:10:truncate.0 00000080:00200000:2.0:1775764853.860440:0:22483:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0xee0:0x0] updating i_size 1048576 i_blocks 2 00000020:00010000:2.0:1775764853.860449:0:22483:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970942c68 (ffff8f994ef582d8/0) at cl_lock_release():245 00000020:00010000:2.0:1775764853.860452:0:22483:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970942c68 (ffff8f994ef582d8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764853.860457:0:22483:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9960102db0 (ffff8f9966dc2008/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764853.860467:0:22483:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.860484:0:22483:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1775764853.860502:0:22483:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9960102df0 (ffff8f9966dc2758/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764853.860508:0:22483:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 4/0,1 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.860517:0:22483:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1775764853.860527:0:22483:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970942c68 (ffff8f994ef582d8/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764853.860530:0:22483:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9960102db0 (ffff8f994ef582d8/1) at cl_lock_fini():72 00000020:00010000:2.0:1775764853.860535:0:22483:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9960102df0 (ffff8f994ef582d8/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764853.860592:0:22483:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xee0:0x0] ignore/verify layout 0/0, layout version 6 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1775764853.860638:0:22483:0:(llite_lib.c:2524:ll_setattr_raw()) COMPLETED file f11a.sanity-pfl:[0x200000401:0xee0:0x0](ffff8f9952eb7448) current size 1048576, valid attrs 168, mode ffff, uid -1734053588, gid -1, new size 1048576, atime 1775764853.000000000, mtime 1775764853.000000000, ctime 1775764853.000000000, rc 0 00000080:00200000:2.0:1775764853.901979:0:22485:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:2.0:1775764853.902009:0:22485:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764853.902028:0:22485:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:2.0:1775764853.902050:0:22485:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:2.0:1775764853.902057:0:22485:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764853.902066:0:22485:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:2.0:1775764853.902071:0:22485:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:2.0:1775764853.902078:0:22485:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac19c7bf8 00010000:00010000:2.0:1775764853.902082:0:22485:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:2.0:1775764853.902091:0:22485:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:2.0:1775764853.902106:0:22485:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:2.0:1775764853.902115:0:22485:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d11a.sanity-pfl, flags=27 00000080:00200000:2.0:1775764853.902121:0:22485:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88), inode mode 41ed mask 1 00000080:00200000:2.0:1775764853.902134:0:22485:0:(namei.c:1399:ll_atomic_open()) VFS Op:name=f11a.sanity-pfl, dir=[0x200000401:0xedf:0x0](ffff8f9952eedb88), file ffff8f99435db380, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1775764853.902141:0:22485:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=f11a.sanity-pfl, dir=[0x200000401:0xedf:0x0](ffff8f9952eedb88), intent=open 00000002:00010000:2.0:1775764853.902155:0:22485:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f11a.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000401:0xedf:0x0], intent: open flags 0304001 00010000:00010000:2.0:1775764853.902470:0:22485:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c99f lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 22485 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.902479:0:22485:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c99f lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 22485 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764853.902492:0:22485:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764853.902506:0:22485:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f99703d3100 x1862022967049984/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1080/132280 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764853.902517:0:22485:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c99f lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 22485 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764853.902541:0:22485:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22485:1862022967049984:192.168.202.146@tcp:101:lfs.0 00000100:00100000:2.0:1775764853.903167:0:22485:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952af3180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764853.910536:0:22485:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22485:1862022967049984:192.168.202.146@tcp:101:lfs.0 00010000:00010000:2.0:1775764853.910560:0:22485:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c99f lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 22485 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.910575:0:22485:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c99f lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 22485 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.910582:0:22485:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c99f lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 22485 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.910588:0:22485:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c99f lrc: 3/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 22485 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.910596:0:22485:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c99f lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 22485 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.910602:0:22485:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99706ea600 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:2.0:1775764853.910607:0:22485:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c99f lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 22485 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.910903:0:22485:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c99f lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 22485 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.910911:0:22485:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c99f lrc: 5/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 22485 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.910919:0:22485:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1775764853.910925:0:22485:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c99f lrc: 3/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 22485 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.910932:0:22485:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c99f lrc: 0/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 22485 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764853.910953:0:22485:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff8f99703d3100 x1862022967049984/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1080/608 e 0 to 0 dl 1775764869 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764853.910969:0:22485:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0xedf:0x0] suppgids 0 -1: rc -20 00000080:00200000:0.0:1775764853.911453:0:22485:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:0.0:1775764853.911476:0:22485:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764853.911493:0:22485:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:0.0:1775764853.911505:0:22485:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:0.0:1775764853.911512:0:22485:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764853.911519:0:22485:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:0.0:1775764853.911523:0:22485:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:0.0:1775764853.911528:0:22485:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac19c7bf8 00010000:00010000:0.0:1775764853.911532:0:22485:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:0.0:1775764853.911540:0:22485:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00200000:0.0:1775764853.911552:0:22485:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:0.0:1775764853.911560:0:22485:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d11a.sanity-pfl, flags=267 00000080:00200000:0.0:1775764853.911567:0:22485:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88), inode mode 41ed mask 44 00000080:00200008:0.0:1775764853.911575:0:22485:0:(file.c:1057:ll_file_open()) START file d11a.sanity-pfl:[0x200000401:0xedf:0x0](ffff8f9952eedb88), flags 304000 00000080:00200080:0.0:1775764853.911614:0:22485:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88) cmd=c0086915 arg=23353d0 00000100:00100000:0.0:1775764853.912062:0:22485:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22485:1862022967050112:192.168.202.146@tcp:34:lfs.0 00000100:00100000:0.0:1775764853.912406:0:22485:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102480 going to sleep for 11 seconds 00000100:00100000:0.0:1775764853.923846:0:22485:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22485:1862022967050112:192.168.202.146@tcp:34:lfs.0 00000080:00200008:0.0:1775764853.924082:0:22485:0:(file.c:447:ll_file_release()) START file d11a.sanity-pfl:[0x200000401:0xedf:0x0](ffff8f9952eedb88), flags 304000 00010000:00010000:0.0:1775764853.924117:0:22485:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640600/0x36f9bdc9e421c967 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc220 expref: -99 pid: 22475 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1775764853.965526:0:22488:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:2.0:1775764853.965556:0:22488:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764853.965575:0:22488:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:2.0:1775764853.965592:0:22488:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00200000:2.0:1775764853.965600:0:22488:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764853.965609:0:22488:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:2.0:1775764853.965612:0:22488:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:2.0:1775764853.965711:0:22488:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac19c7bf8 00010000:00010000:2.0:1775764853.965956:0:22488:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:2.0:1775764853.965969:0:22488:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00000080:00200000:2.0:1775764853.965990:0:22488:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:2.0:1775764853.965999:0:22488:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d11a.sanity-pfl, flags=27 00000080:00200000:2.0:1775764853.966005:0:22488:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88), inode mode 41ed mask 1 00000080:00200000:2.0:1775764853.966018:0:22488:0:(namei.c:1399:ll_atomic_open()) VFS Op:name=f11a.sanity-pfl, dir=[0x200000401:0xedf:0x0](ffff8f9952eedb88), file ffff8f9971135980, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1775764853.966025:0:22488:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=f11a.sanity-pfl, dir=[0x200000401:0xedf:0x0](ffff8f9952eedb88), intent=open 00000002:00010000:2.0:1775764853.966042:0:22488:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f11a.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000401:0xedf:0x0], intent: open flags 0304001 00010000:00010000:2.0:1775764853.966101:0:22488:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c9a6 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 22488 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.966110:0:22488:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c9a6 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 22488 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764853.966122:0:22488:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764853.966135:0:22488:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f99703d3100 x1862022967050368/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1080/132280 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764853.966144:0:22488:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c9a6 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 22488 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764853.966167:0:22488:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22488:1862022967050368:192.168.202.146@tcp:101:lfs.0 00000100:00100000:2.0:1775764853.967201:0:22488:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952af3180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764853.974079:0:22488:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22488:1862022967050368:192.168.202.146@tcp:101:lfs.0 00010000:00010000:2.0:1775764853.974104:0:22488:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c9a6 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 22488 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.974119:0:22488:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c9a6 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 22488 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.974132:0:22488:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c9a6 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 22488 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.974139:0:22488:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c9a6 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 22488 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.974148:0:22488:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c9a6 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 22488 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.974155:0:22488:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99706ea600 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:2.0:1775764853.974161:0:22488:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c9a6 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 22488 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.974168:0:22488:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c9a6 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 22488 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.974186:0:22488:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c9a6 lrc: 5/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 22488 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.974194:0:22488:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1775764853.974200:0:22488:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c9a6 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 22488 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.974206:0:22488:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c9a6 lrc: 0/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 22488 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764853.974225:0:22488:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff8f99703d3100 x1862022967050368/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1080/608 e 0 to 0 dl 1775764869 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764853.974242:0:22488:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0xedf:0x0] suppgids 0 -1: rc -20 00000080:00200000:2.0:1775764853.974398:0:22488:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:2.0:1775764853.974419:0:22488:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764853.974434:0:22488:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00010000:00010000:2.0:1775764853.974446:0:22488:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00000080:00200000:2.0:1775764853.974453:0:22488:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764853.974459:0:22488:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:2.0:1775764853.974463:0:22488:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:2.0:1775764853.974467:0:22488:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac19c7bf8 00010000:00010000:2.0:1775764853.974471:0:22488:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00010000:00010000:2.0:1775764853.974478:0:22488:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00000080:00200000:2.0:1775764853.974492:0:22488:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:2.0:1775764853.974499:0:22488:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d11a.sanity-pfl, flags=267 00000080:00200000:2.0:1775764853.974505:0:22488:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88), inode mode 41ed mask 44 00000080:00200008:2.0:1775764853.974515:0:22488:0:(file.c:1057:ll_file_open()) START file d11a.sanity-pfl:[0x200000401:0xedf:0x0](ffff8f9952eedb88), flags 304000 00000080:00200080:2.0:1775764853.974552:0:22488:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88) cmd=c0086915 arg=1a6d3d0 00000100:00100000:2.0:1775764853.974605:0:22488:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22488:1862022967050496:192.168.202.146@tcp:34:lfs.0 00000100:00100000:2.0:1775764853.974687:0:22488:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952af3180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764853.981368:0:22488:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22488:1862022967050496:192.168.202.146@tcp:34:lfs.0 00000080:00200008:2.0:1775764853.981579:0:22488:0:(file.c:447:ll_file_release()) START file d11a.sanity-pfl:[0x200000401:0xedf:0x0](ffff8f9952eedb88), flags 304000 00010000:00010000:2.0:1775764853.981610:0:22488:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640600/0x36f9bdc9e421c967 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc220 expref: -99 pid: 22475 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:3.0:1775764854.024574:0:22491:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:3.0:1775764854.024604:0:22491:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764854.027125:0:22491:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00010000:00010000:3.0:1775764854.027149:0:22491:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00000080:00200000:3.0:1775764854.027160:0:22491:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764854.027170:0:22491:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:3.0:1775764854.027308:0:22491:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:3.0:1775764854.027319:0:22491:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac19c7bf8 00010000:00010000:3.0:1775764854.027323:0:22491:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00010000:00010000:3.0:1775764854.027332:0:22491:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00000080:00200000:3.0:1775764854.027350:0:22491:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:3.0:1775764854.027401:0:22491:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d11a.sanity-pfl, flags=27 00000080:00200000:3.0:1775764854.027408:0:22491:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764854.027425:0:22491:0:(namei.c:1399:ll_atomic_open()) VFS Op:name=f11a.sanity-pfl, dir=[0x200000401:0xedf:0x0](ffff8f9952eedb88), file ffff8f9943605200, open_flags 18800, mode 0 opened 0 00000080:00200000:3.0:1775764854.027432:0:22491:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=f11a.sanity-pfl, dir=[0x200000401:0xedf:0x0](ffff8f9952eedb88), intent=open 00000002:00010000:3.0:1775764854.027451:0:22491:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f11a.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000401:0xedf:0x0], intent: open flags 0304001 00010000:00010000:3.0:1775764854.027498:0:22491:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c9ad lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 22491 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764854.027506:0:22491:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c9ad lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 22491 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:3.0:1775764854.027516:0:22491:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1775764854.027527:0:22491:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970edea00 x1862022967050752/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1080/132280 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1775764854.027535:0:22491:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c9ad lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 22491 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764854.027557:0:22491:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22491:1862022967050752:192.168.202.146@tcp:101:lfs.0 00000100:00100000:3.0:1775764854.028147:0:22491:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994ecbbc00 going to sleep for 11 seconds 00000100:00100000:3.0:1775764854.037088:0:22491:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22491:1862022967050752:192.168.202.146@tcp:101:lfs.0 00010000:00010000:3.0:1775764854.037112:0:22491:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c9ad lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 22491 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764854.037125:0:22491:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c9ad lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 22491 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764854.037131:0:22491:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c9ad lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 22491 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764854.037313:0:22491:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c9ad lrc: 3/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 22491 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764854.037322:0:22491:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c9ad lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 22491 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764854.037328:0:22491:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99435d4200 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:3.0:1775764854.037333:0:22491:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c9ad lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 22491 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764854.037340:0:22491:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c9ad lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 22491 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764854.037348:0:22491:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c9ad lrc: 5/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 22491 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764854.037356:0:22491:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1775764854.037362:0:22491:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c9ad lrc: 3/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 22491 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764854.037367:0:22491:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c9ad lrc: 0/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 22491 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764854.037387:0:22491:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff8f9970edea00 x1862022967050752/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1080/608 e 0 to 0 dl 1775764870 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1775764854.037402:0:22491:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0xedf:0x0] suppgids 0 -1: rc -20 00000080:00200000:3.0:1775764854.037554:0:22491:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:3.0:1775764854.037575:0:22491:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764854.037588:0:22491:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00010000:00010000:3.0:1775764854.037626:0:22491:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00000080:00200000:3.0:1775764854.037634:0:22491:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764854.037639:0:22491:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:3.0:1775764854.037642:0:22491:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:3.0:1775764854.037647:0:22491:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac19c7bf8 00010000:00010000:3.0:1775764854.037650:0:22491:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00010000:00010000:3.0:1775764854.037657:0:22491:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00000080:00200000:3.0:1775764854.037669:0:22491:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:3.0:1775764854.037675:0:22491:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d11a.sanity-pfl, flags=267 00000080:00200000:3.0:1775764854.037682:0:22491:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88), inode mode 41ed mask 44 00000080:00200008:3.0:1775764854.037690:0:22491:0:(file.c:1057:ll_file_open()) START file d11a.sanity-pfl:[0x200000401:0xedf:0x0](ffff8f9952eedb88), flags 304000 00000080:00200080:3.0:1775764854.037977:0:22491:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88) cmd=c0086915 arg=1caa3d0 00000100:00100000:3.0:1775764854.038043:0:22491:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22491:1862022967050880:192.168.202.146@tcp:34:lfs.0 00000100:00100000:3.0:1775764854.038196:0:22491:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994ecbbc00 going to sleep for 11 seconds 00000100:00100000:3.0:1775764854.044505:0:22491:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22491:1862022967050880:192.168.202.146@tcp:34:lfs.0 00000080:00200008:3.0:1775764854.047604:0:22491:0:(file.c:447:ll_file_release()) START file d11a.sanity-pfl:[0x200000401:0xedf:0x0](ffff8f9952eedb88), flags 304000 00010000:00010000:3.0:1775764854.047769:0:22491:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640600/0x36f9bdc9e421c967 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc220 expref: -99 pid: 22475 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:0.0:1775764854.066323:0:22493:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:0.0:1775764854.066350:0:22493:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764854.066366:0:22493:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00010000:00010000:0.0:1775764854.066381:0:22493:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00000080:00200000:0.0:1775764854.066389:0:22493:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764854.066396:0:22493:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:0.0:1775764854.066400:0:22493:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:0.0:1775764854.066407:0:22493:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac1e03c08 00010000:00010000:0.0:1775764854.066411:0:22493:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00010000:00010000:0.0:1775764854.066419:0:22493:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00000080:00200000:0.0:1775764854.066435:0:22493:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:0.0:1775764854.066443:0:22493:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d11a.sanity-pfl, flags=25 00000080:00200000:0.0:1775764854.066448:0:22493:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764854.066490:0:22493:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=f11a.sanity-pfl, dir=[0x200000401:0xedf:0x0](ffff8f9952eedb88), flags=9 00000080:00200000:0.0:1775764854.066494:0:22493:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=f11a.sanity-pfl, dir=[0x200000401:0xedf:0x0](ffff8f9952eedb88), intent=getattr 00000002:00010000:0.0:1775764854.066509:0:22493:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f11a.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000401:0xedf:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764854.066561:0:22493:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c9b4 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 22493 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.066568:0:22493:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c9b4 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 22493 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.066578:0:22493:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c9b4 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 22493 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764854.066603:0:22493:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job truncate:3f83546c-01a9-4156-8a97-44c446245d3e:22493:1862022967051008:192.168.202.146@tcp:101:truncate.0 00000100:00100000:0.0:1775764854.067112:0:22493:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102480 going to sleep for 11 seconds 00000100:00100000:0.0:1775764854.073055:0:22493:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job truncate:3f83546c-01a9-4156-8a97-44c446245d3e:22493:1862022967051008:192.168.202.146@tcp:101:truncate.0 00010000:00010000:0.0:1775764854.073079:0:22493:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c9b4 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc370 expref: -99 pid: 22493 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.073098:0:22493:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c9b4 lrc: 4/1,0 mode: --/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc370 expref: -99 pid: 22493 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.073105:0:22493:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c9b4 lrc: 4/1,0 mode: --/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc370 expref: -99 pid: 22493 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.073118:0:22493:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c9b4 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc370 expref: -99 pid: 22493 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.073129:0:22493:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c9b4 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc370 expref: -99 pid: 22493 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.073164:0:22493:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c9b4 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc370 expref: -99 pid: 22493 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.073309:0:22493:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c9b4 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc370 expref: -99 pid: 22493 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.073316:0:22493:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c9b4 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc370 expref: -99 pid: 22493 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764854.073327:0:22493:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970a75180 x1862022967051008/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/1136 e 0 to 0 dl 1775764870 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'truncate.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1775764854.073336:0:22493:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c9b4 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc370 expref: -99 pid: 22493 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:0.0:1775764854.073349:0:22493:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c9b4 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc370 expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.073358:0:22493:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/3808 (0 0) 00000080:00200000:0.0:1775764854.073364:0:22493:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0xedf:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1775764854.073464:0:22493:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0xee0:0x0](ffff8f9952eb7448): rc = 0 00000080:00200000:0.0:1775764854.073487:0:22493:0:(file.c:6766:ll_layout_conf()) [0x200000401:0xee0:0x0]: layout version change: 6 -> 6 00000080:00010000:0.0:1775764854.073498:0:22493:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xee0:0x0](ffff8f9952eb7448) for lock 0x36f9bdc9e421c9b4 00000080:00010000:0.0:1775764854.073510:0:22493:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xee0:0x0](ffff8f9952eb7448) 00000080:00010000:0.0:1775764854.073513:0:22493:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c9b4 from it ffffafbac1e03bd0 00010000:00010000:0.0:1775764854.073517:0:22493:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c9b4 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fc370 expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.073545:0:22493:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c9b4 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fc370 expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764854.073586:0:22493:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xee0:0x0](ffff8f9952eb7448), inode mode 81a4 mask 2 00000080:00200008:0.0:1775764854.073598:0:22493:0:(llite_lib.c:2274:ll_setattr_raw()) START file f11a.sanity-pfl:[0x200000401:0xee0:0x0](ffff8f9952eb7448) current size 1048576, valid attrs 8, mode ffff, uid -1734053588, gid -1, new size 3145727, atime 1775764854.000000000, mtime 1775764854.000000000, ctime 1775764854.000000000 00010000:00010000:0.0:1775764854.074065:0:22493:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c9b4 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5fc370 expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764854.074579:0:22493:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0xee0:0x0]: losing layout lock 00010000:00010000:0.0:1775764854.075358:0:22493:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c98a lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5fc2c8 expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1775764854.075398:0:22493:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0xee0:0x0]: losing layout lock 00010000:00010000:0.0:1775764854.075450:0:22493:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c9b4 lrc: 1/0,0 mode: --/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fc370 expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764854.075457:0:22493:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c98a lrc: 1/0,0 mode: --/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fc2c8 expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775764854.075463:0:22493:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1775764854.075466:0:22493:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821ccc00/0x36f9bdc9e421c9b4 lrc: 0/0,0 mode: --/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fc370 expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764854.075483:0:22493:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706eb800/0x36f9bdc9e421c98a lrc: 0/0,0 mode: --/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fc2c8 expref: -99 pid: 22483 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000020:00100000:0.0:1775764854.075546:0:22493:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1775764854.075561:0:22493:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970a75180 x1862022967051264/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764854.075589:0:22493:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job truncate:3f83546c-01a9-4156-8a97-44c446245d3e:22493:1862022967051264:192.168.202.146@tcp:36:truncate.0 00000100:00100000:0.0:1775764854.076980:0:22493:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102480 going to sleep for 11 seconds 00000100:00100000:0.0:1775764854.084498:0:22493:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job truncate:3f83546c-01a9-4156-8a97-44c446245d3e:22493:1862022967051264:192.168.202.146@tcp:36:truncate.0 00000080:00200000:0.0:1775764854.084578:0:22493:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xee0:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00010000:0.0:1775764854.084601:0:22493:0:(file.c:6988:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200000401:0xee0:0x0](ffff8f9952eb7448) 00000002:00010000:0.0:1775764854.084613:0:22493:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xee0:0x0]) in obj [0x200000401:0xee0:0x0], intent: layout flags 0 00010000:00010000:0.0:1775764854.084680:0:22493:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd600/0x36f9bdc9e421c9bb lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 22493 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.084691:0:22493:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd600/0x36f9bdc9e421c9bb lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 22493 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.084701:0:22493:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd600/0x36f9bdc9e421c9bb lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 22493 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764854.084728:0:22493:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job truncate:3f83546c-01a9-4156-8a97-44c446245d3e:22493:1862022967051392:192.168.202.146@tcp:101:truncate.0 00000100:00100000:0.0:1775764854.084817:0:22493:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102480 going to sleep for 11 seconds 00000100:00100000:0.0:1775764854.090314:0:22493:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job truncate:3f83546c-01a9-4156-8a97-44c446245d3e:22493:1862022967051392:192.168.202.146@tcp:101:truncate.0 00010000:00010000:0.0:1775764854.090345:0:22493:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd600/0x36f9bdc9e421c9bb lrc: 4/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc37e expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.090358:0:22493:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd600/0x36f9bdc9e421c9bb lrc: 4/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc37e expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764854.090410:0:22493:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8f9970a74380 x1862022967051392/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 376/896 e 0 to 0 dl 1775764870 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'truncate.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1775764854.090422:0:22493:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd600/0x36f9bdc9e421c9bb lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc37e expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:0.0:1775764854.090435:0:22493:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd600/0x36f9bdc9e421c9bb lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc37e expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.090448:0:22493:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 16 res 8589935617/3808 (0 0) 00000080:00200000:0.0:1775764854.090454:0:22493:0:(namei.c:943:ll_intent_lock()) intent lock 1024 on i1 [0x200000401:0xee0:0x0] suppgids 0 0: rc 0 00000080:00010000:0.0:1775764854.090473:0:22493:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xee0:0x0](ffff8f9952eb7448) for lock 0x36f9bdc9e421c9bb 00000080:00010000:0.0:1775764854.090481:0:22493:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c9bb from it ffffafbac1e03ad8 00010000:00010000:0.0:1775764854.090484:0:22493:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd600/0x36f9bdc9e421c9bb lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc37e expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.090493:0:22493:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd600/0x36f9bdc9e421c9bb lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc37e expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764854.090514:0:22493:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd600/0x36f9bdc9e421c9bb lrc: 2/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc37e expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764854.090525:0:22493:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd600/0x36f9bdc9e421c9bb lrc: 2/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc37e expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:0.0:1775764854.090531:0:22493:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200000401:0xee0:0x0](000000002c188e9c) being reconfigured ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd600/0x36f9bdc9e421c9bb lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc37e expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764854.090641:0:22493:0:(file.c:6766:ll_layout_conf()) [0x200000401:0xee0:0x0]: layout version change: 4294967294 -> 6 00010000:00010000:0.0:1775764854.090662:0:22493:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd600/0x36f9bdc9e421c9bb lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fc37e expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764854.090669:0:22493:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd600/0x36f9bdc9e421c9bb lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fc37e expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764854.090685:0:22493:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xee0:0x0] ignore/verify layout 0/0, layout version 6 need write layout 1, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764854.090690:0:22493:0:(vvp_io.c:355:vvp_io_fini()) [0x200000401:0xee0:0x0] write layout, type 3 [0x0, 0x300000) 00000080:00010000:0.0:1775764854.090697:0:22493:0:(file.c:6988:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200000401:0xee0:0x0](ffff8f9952eb7448) 00000002:00010000:0.0:1775764854.090705:0:22493:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xee0:0x0]) in obj [0x200000401:0xee0:0x0], intent: layout flags 02 00010000:00010000:0.0:1775764854.090728:0:22493:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd600/0x36f9bdc9e421c9bb lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5fc37e expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764854.090790:0:22493:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0xee0:0x0]: losing layout lock 00010000:00010000:0.0:1775764854.090825:0:22493:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd600/0x36f9bdc9e421c9bb lrc: 1/0,0 mode: --/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fc37e expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764854.090832:0:22493:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1775764854.090835:0:22493:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd600/0x36f9bdc9e421c9bb lrc: 0/0,0 mode: --/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fc37e expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764854.090876:0:22493:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(EX) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c9c2 lrc: 3/0,1 mode: --/EX res: [0x200000401:0xee0:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 22493 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.090882:0:22493:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c9c2 lrc: 3/0,1 mode: --/EX res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 22493 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1775764854.090914:0:22493:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1775764854.090921:0:22493:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970a74380 x1862022967051648/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 376/896 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1775764854.090931:0:22493:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c9c2 lrc: 3/0,1 mode: --/EX res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 22493 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764854.090951:0:22493:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job truncate:3f83546c-01a9-4156-8a97-44c446245d3e:22493:1862022967051648:192.168.202.146@tcp:101:truncate.0 00000100:00100000:0.0:1775764854.091018:0:22493:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102480 going to sleep for 11 seconds 00000100:00100000:0.0:1775764854.097396:0:22493:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job truncate:3f83546c-01a9-4156-8a97-44c446245d3e:22493:1862022967051648:192.168.202.146@tcp:101:truncate.0 00010000:00010000:0.0:1775764854.097424:0:22493:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode CR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c9c2 lrc: 4/0,1 mode: --/EX res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc38c expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.097439:0:22493:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c9c2 lrc: 4/0,1 mode: --/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc38c expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.097452:0:22493:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c9c2 lrc: 4/0,1 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc38c expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.097463:0:22493:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c9c2 lrc: 4/0,1 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc38c expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.097478:0:22493:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c9c2 lrc: 5/1,1 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc38c expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.097487:0:22493:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(EX) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c9c2 lrc: 5/1,1 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc38c expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.097540:0:22493:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c9c2 lrc: 4/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc38c expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764854.097552:0:22493:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8f9970a74380 x1862022967051648/t8589934625(8589934625) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 376/896 e 0 to 0 dl 1775764870 ref 2 fl Complete:RQU/604/0 rc 0/0 job:'truncate.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1775764854.097570:0:22493:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c9c2 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc38c expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:0.0:1775764854.097581:0:22493:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c9c2 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc38c expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.097593:0:22493:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 16 res 8589935617/3808 (0 0) 00000080:00200000:0.0:1775764854.097599:0:22493:0:(namei.c:943:ll_intent_lock()) intent lock 1024 on i1 [0x200000401:0xee0:0x0] suppgids 0 0: rc 0 00000080:00010000:0.0:1775764854.097606:0:22493:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xee0:0x0](ffff8f9952eb7448) for lock 0x36f9bdc9e421c9c2 00000080:00010000:0.0:1775764854.097631:0:22493:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c9c2 from it ffffafbac1e03b40 00010000:00010000:0.0:1775764854.097635:0:22493:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c9c2 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc38c expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.097643:0:22493:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c9c2 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc38c expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764854.097665:0:22493:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xee0:0x0] ignore/verify layout 0/0, layout version 6 restore needed 0 00010000:00010000:0.0:1775764854.097678:0:22493:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c9c2 lrc: 2/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc38c expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764854.097690:0:22493:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c9c2 lrc: 2/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc38c expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:0.0:1775764854.097712:0:22493:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200000401:0xee0:0x0](000000002c188e9c) being reconfigured ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c9c2 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc38c expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764854.097872:0:22493:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c9c2 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc38c expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764854.097880:0:22493:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c9c2 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc38c expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764854.097910:0:22493:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xee0:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764854.097920:0:22493:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1775764854.097941:0:22493:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:0.0:1775764854.097956:0:22493:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00020000:00200000:0.0:1775764854.097958:0:22493:0:(lov_io.c:860:lov_io_iter_init()) component[3] flags 0x0 00020000:00200000:0.0:1775764854.097959:0:22493:0:(lov_io.c:860:lov_io_iter_init()) component[4] flags 0x0 00000080:00200000:0.0:1775764854.097992:0:22493:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xee0:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1775764854.098014:0:22493:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c9c2 lrc: 2/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc38c expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764854.098023:0:22493:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c9c2 lrc: 2/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc38c expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:0.0:1775764854.098030:0:22493:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200000401:0xee0:0x0](000000002c188e9c) being reconfigured ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c9c2 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc38c expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764854.098063:0:22493:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xee0:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764854.098068:0:22493:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1775764854.098077:0:22493:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:0.0:1775764854.098110:0:22493:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00020000:00200000:0.0:1775764854.098112:0:22493:0:(lov_io.c:860:lov_io_iter_init()) component[3] flags 0x0 00020000:00200000:0.0:1775764854.098113:0:22493:0:(lov_io.c:860:lov_io_iter_init()) component[4] flags 0x0 00000080:00200000:0.0:1775764854.098132:0:22493:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xee0:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:0.0:1775764854.098150:0:22493:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 0 blocks 1 [cma]time 0 0 0 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 1/0,0 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:0.0:1775764854.098175:0:22493:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 1048576 blocks 1 [cma]time 1775764853 1775764853 0 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 1/0,0 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764854.098248:0:22493:0:(file.c:6766:ll_layout_conf()) [0x200000401:0xee0:0x0]: layout version change: 4294967294 -> 7 00010000:00010000:0.0:1775764854.098259:0:22493:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c9c2 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fc38c expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764854.098267:0:22493:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c9c2 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fc38c expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00020000:00200000:0.0:1775764854.098282:0:22493:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x10 00020000:00200000:0.0:1775764854.098295:0:22493:0:(lov_io.c:860:lov_io_iter_init()) component[3] flags 0x0 00020000:00200000:0.0:1775764854.098297:0:22493:0:(lov_io.c:860:lov_io_iter_init()) component[4] flags 0x0 00000080:00200000:0.0:1775764854.098300:0:22493:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:0.0:1775764854.098321:0:22493:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764854.098326:0:22493:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f9952395b30, osc lock ffff8f996686ef70, flags 0x0 00000008:00010000:0.0:1775764854.098335:0:22493:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764854.098337:0:22493:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f9952395b70, osc lock ffff8f996686e260, flags 0x0 00000008:00010000:0.0:1775764854.098340:0:22493:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764854.098342:0:22493:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f9952395bb0, osc lock ffff8f996686ebe0, flags 0x0 00010000:00010000:0.0:1775764854.098356:0:22493:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764854.098385:0:22493:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764854.098396:0:22493:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 6/0,2 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:0.0:1775764854.098405:0:22493:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 5/0,2 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764854.098413:0:22493:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 6/0,2 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764854.098419:0:22493:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764854.098429:0:22493:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764854.098438:0:22493:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (1048576 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764854.098445:0:22493:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 6/0,2 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:0.0:1775764854.098470:0:22493:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 1048576, setting rss=1048576; kms=1048576, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 5/0,2 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764854.098478:0:22493:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 6/0,2 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764854.098484:0:22493:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764854.098544:0:22493:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 3/0,1 mode: --/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.098553:0:22493:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 3/0,1 mode: --/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [2097152->3145727] (req 2097152->3145727) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.098562:0:22493:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 3/0,1 mode: --/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [2097152->3145727] (req 2097152->3145727) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764854.098587:0:22493:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job truncate:3f83546c-01a9-4156-8a97-44c446245d3e:22493:1862022967051776:192.168.202.146@tcp:101:truncate.0 00000100:00100000:0.0:1775764854.098674:0:22493:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102480 going to sleep for 11 seconds 00000100:00100000:0.0:1775764854.102607:0:22493:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job truncate:3f83546c-01a9-4156-8a97-44c446245d3e:22493:1862022967051776:192.168.202.146@tcp:101:truncate.0 00010000:00010000:0.0:1775764854.102650:0:22493:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 4/0,1 mode: --/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.102724:0:22493:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.102770:0:22493:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.102788:0:22493:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 5/0,2 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1775764854.102802:0:22493:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 4/0,2 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.102814:0:22493:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 5/0,2 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.102822:0:22493:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764854.102927:0:2380:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1775764854.102944:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967051904:192.168.202.146@tcp:10:truncate.0 00000100:00100000:0.0:1775764854.108093:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967051904:192.168.202.146@tcp:10:truncate.0 00000080:00200000:2.0:1775764854.108543:0:22493:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0xee0:0x0] updating i_size 3145727 i_blocks 3 00000020:00010000:2.0:1775764854.108553:0:22493:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99713e94e8 (ffff8f9966dc21b8/0) at cl_lock_release():245 00000020:00010000:2.0:1775764854.108556:0:22493:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99713e94e8 (ffff8f9966dc21b8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764854.108561:0:22493:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9952395b30 (ffff8f9966dc2e18/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764854.108573:0:22493:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764854.108590:0:22493:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00010000:2.0:1775764854.108607:0:22493:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9952395b70 (ffff8f9966dc2878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764854.108623:0:22493:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 4/0,1 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764854.108633:0:22493:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00010000:2.0:1775764854.108642:0:22493:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9952395bb0 (ffff8f9966dc2b48/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764854.108646:0:22493:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764854.108652:0:22493:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1775764854.108658:0:22493:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99713e94e8 (ffff8f9966dc21b8/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764854.108662:0:22493:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9952395b30 (ffff8f9966dc21b8/1) at cl_lock_fini():72 00000020:00010000:2.0:1775764854.108667:0:22493:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9952395b70 (ffff8f9966dc21b8/1) at cl_lock_fini():72 00000020:00010000:2.0:1775764854.108670:0:22493:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9952395bb0 (ffff8f9966dc21b8/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764854.108760:0:22493:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xee0:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1775764854.108783:0:22493:0:(llite_lib.c:2524:ll_setattr_raw()) COMPLETED file f11a.sanity-pfl:[0x200000401:0xee0:0x0](ffff8f9952eb7448) current size 3145727, valid attrs 168, mode ffff, uid -1734053588, gid -1, new size 3145727, atime 1775764854.000000000, mtime 1775764854.000000000, ctime 1775764854.000000000, rc 0 00000080:00200000:3.0:1775764854.121725:0:22494:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:3.0:1775764854.122002:0:22494:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764854.122025:0:22494:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00010000:00010000:3.0:1775764854.122042:0:22494:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00000080:00200000:3.0:1775764854.122049:0:22494:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764854.122058:0:22494:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:3.0:1775764854.122061:0:22494:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:3.0:1775764854.122067:0:22494:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac1d7bc08 00010000:00010000:3.0:1775764854.122071:0:22494:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00010000:00010000:3.0:1775764854.122079:0:22494:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 21 lru_type: 1 00000080:00200000:3.0:1775764854.122094:0:22494:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:3.0:1775764854.122102:0:22494:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d11a.sanity-pfl, flags=25 00000080:00200000:3.0:1775764854.122107:0:22494:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764854.122118:0:22494:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=f11a.sanity-pfl, dir=[0x200000401:0xedf:0x0](ffff8f9952eedb88), flags=9 00000080:00200000:3.0:1775764854.122122:0:22494:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=f11a.sanity-pfl, dir=[0x200000401:0xedf:0x0](ffff8f9952eedb88), intent=getattr 00000002:00010000:3.0:1775764854.122132:0:22494:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f11a.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000401:0xedf:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764854.122334:0:22494:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c9d0 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 22494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764854.122369:0:22494:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c9d0 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 22494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764854.122379:0:22494:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c9d0 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 22494 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764854.122407:0:22494:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job truncate:3f83546c-01a9-4156-8a97-44c446245d3e:22494:1862022967052032:192.168.202.146@tcp:101:truncate.0 00000100:00100000:3.0:1775764854.122560:0:22494:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994ecbbc00 going to sleep for 11 seconds 00000100:00100000:3.0:1775764854.133450:0:22494:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job truncate:3f83546c-01a9-4156-8a97-44c446245d3e:22494:1862022967052032:192.168.202.146@tcp:101:truncate.0 00010000:00010000:3.0:1775764854.133477:0:22494:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c9d0 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc3af expref: -99 pid: 22494 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764854.133494:0:22494:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c9d0 lrc: 4/1,0 mode: --/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc3af expref: -99 pid: 22494 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764854.133502:0:22494:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c9d0 lrc: 4/1,0 mode: --/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc3af expref: -99 pid: 22494 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764854.133513:0:22494:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c9d0 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc3af expref: -99 pid: 22494 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764854.133524:0:22494:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c9d0 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc3af expref: -99 pid: 22494 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764854.133535:0:22494:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c9d0 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc3af expref: -99 pid: 22494 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764854.133579:0:22494:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c9d0 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc3af expref: -99 pid: 22494 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764854.133587:0:22494:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c9d0 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc3af expref: -99 pid: 22494 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764854.133598:0:22494:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970edea00 x1862022967052032/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/1136 e 0 to 0 dl 1775764870 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'truncate.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1775764854.133610:0:22494:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c9d0 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc3af expref: -99 pid: 22494 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:3.0:1775764854.133623:0:22494:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c9d0 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc3af expref: -99 pid: 22494 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764854.133633:0:22494:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/3808 (0 0) 00000080:00200000:3.0:1775764854.133639:0:22494:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0xedf:0x0] suppgids 0 -1: rc 0 00000080:00200000:3.0:1775764854.133806:0:22494:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0xee0:0x0](ffff8f9952eb7448): rc = 0 00000080:00200000:3.0:1775764854.133831:0:22494:0:(file.c:6766:ll_layout_conf()) [0x200000401:0xee0:0x0]: layout version change: 7 -> 7 00000080:00010000:3.0:1775764854.133841:0:22494:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xee0:0x0](ffff8f9952eb7448) for lock 0x36f9bdc9e421c9d0 00000080:00010000:3.0:1775764854.133862:0:22494:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xee0:0x0](ffff8f9952eb7448) 00000080:00010000:3.0:1775764854.133866:0:22494:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c9d0 from it ffffafbac1d7bbd0 00010000:00010000:3.0:1775764854.133869:0:22494:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c9d0 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fc3af expref: -99 pid: 22494 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764854.133878:0:22494:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c9d0 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fc3af expref: -99 pid: 22494 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764854.133932:0:22494:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xee0:0x0](ffff8f9952eb7448), inode mode 81a4 mask 2 00000080:00200008:3.0:1775764854.133945:0:22494:0:(llite_lib.c:2274:ll_setattr_raw()) START file f11a.sanity-pfl:[0x200000401:0xee0:0x0](ffff8f9952eb7448) current size 3145727, valid attrs 8, mode ffff, uid -1734053588, gid -1, new size 1048577, atime 1775764854.000000000, mtime 1775764854.000000000, ctime 1775764854.000000000 00010000:00010000:3.0:1775764854.133968:0:22494:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c9d0 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5fc3af expref: -99 pid: 22494 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764854.133991:0:22494:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0xee0:0x0]: losing layout lock 00010000:00010000:3.0:1775764854.134009:0:22494:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c9c2 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5fc38c expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764854.134023:0:22494:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0xee0:0x0]: losing layout lock 00010000:00010000:3.0:1775764854.134055:0:22494:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c9d0 lrc: 1/0,0 mode: --/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fc3af expref: -99 pid: 22494 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764854.134062:0:22494:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c9c2 lrc: 1/0,0 mode: --/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fc38c expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1775764854.134067:0:22494:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1775764854.134070:0:22494:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4200/0x36f9bdc9e421c9d0 lrc: 0/0,0 mode: --/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fc3af expref: -99 pid: 22494 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764854.134081:0:22494:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cce00/0x36f9bdc9e421c9c2 lrc: 0/0,0 mode: --/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fc38c expref: -99 pid: 22493 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000020:00100000:3.0:1775764854.134101:0:22494:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1775764854.134109:0:22494:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970edea00 x1862022967052288/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775764854.134148:0:22494:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job truncate:3f83546c-01a9-4156-8a97-44c446245d3e:22494:1862022967052288:192.168.202.146@tcp:36:truncate.0 00000100:00100000:2.0:1775764854.138900:0:22494:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994ecbbc00 going to sleep for 11 seconds 00000100:00100000:0.0:1775764854.157578:0:22494:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job truncate:3f83546c-01a9-4156-8a97-44c446245d3e:22494:1862022967052288:192.168.202.146@tcp:36:truncate.0 00000080:00200000:0.0:1775764854.157831:0:22494:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xee0:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00010000:0.0:1775764854.157879:0:22494:0:(file.c:6988:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200000401:0xee0:0x0](ffff8f9952eb7448) 00000002:00010000:0.0:1775764854.157891:0:22494:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xee0:0x0]) in obj [0x200000401:0xee0:0x0], intent: layout flags 0 00010000:00010000:0.0:1775764854.157945:0:22494:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c9d7 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 22494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.157955:0:22494:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c9d7 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 22494 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.157964:0:22494:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c9d7 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 22494 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764854.157984:0:22494:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job truncate:3f83546c-01a9-4156-8a97-44c446245d3e:22494:1862022967052416:192.168.202.146@tcp:101:truncate.0 00000100:00100000:0.0:1775764854.158131:0:22494:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102480 going to sleep for 11 seconds 00000100:00100000:0.0:1775764854.165638:0:22494:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job truncate:3f83546c-01a9-4156-8a97-44c446245d3e:22494:1862022967052416:192.168.202.146@tcp:101:truncate.0 00010000:00010000:0.0:1775764854.165668:0:22494:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c9d7 lrc: 4/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc3bd expref: -99 pid: 22494 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.165682:0:22494:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c9d7 lrc: 4/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc3bd expref: -99 pid: 22494 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764854.165695:0:22494:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8f9970a75500 x1862022967052416/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 376/896 e 0 to 0 dl 1775764870 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'truncate.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1775764854.165706:0:22494:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c9d7 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc3bd expref: -99 pid: 22494 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:0.0:1775764854.165719:0:22494:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c9d7 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc3bd expref: -99 pid: 22494 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.165806:0:22494:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 16 res 8589935617/3808 (0 0) 00000080:00200000:0.0:1775764854.165812:0:22494:0:(namei.c:943:ll_intent_lock()) intent lock 1024 on i1 [0x200000401:0xee0:0x0] suppgids 0 0: rc 0 00000080:00010000:0.0:1775764854.165829:0:22494:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xee0:0x0](ffff8f9952eb7448) for lock 0x36f9bdc9e421c9d7 00000080:00010000:0.0:1775764854.165836:0:22494:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c9d7 from it ffffafbac1d7bad8 00010000:00010000:0.0:1775764854.165840:0:22494:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c9d7 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc3bd expref: -99 pid: 22494 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.165860:0:22494:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c9d7 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc3bd expref: -99 pid: 22494 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764854.165883:0:22494:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c9d7 lrc: 2/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc3bd expref: -99 pid: 22494 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764854.165894:0:22494:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c9d7 lrc: 2/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc3bd expref: -99 pid: 22494 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:0.0:1775764854.165902:0:22494:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200000401:0xee0:0x0](000000002c188e9c) being reconfigured ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c9d7 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc3bd expref: -99 pid: 22494 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764854.165972:0:22494:0:(file.c:6766:ll_layout_conf()) [0x200000401:0xee0:0x0]: layout version change: 4294967294 -> 7 00010000:00010000:0.0:1775764854.165990:0:22494:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c9d7 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fc3bd expref: -99 pid: 22494 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764854.165998:0:22494:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c9d7 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fc3bd expref: -99 pid: 22494 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00020000:00200000:0.0:1775764854.166018:0:22494:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:0.0:1775764854.166059:0:22494:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x10 00020000:00200000:0.0:1775764854.166086:0:22494:0:(lov_io.c:860:lov_io_iter_init()) component[3] flags 0x0 00020000:00200000:0.0:1775764854.166088:0:22494:0:(lov_io.c:860:lov_io_iter_init()) component[4] flags 0x0 00000080:00200000:0.0:1775764854.166091:0:22494:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:0.0:1775764854.166115:0:22494:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764854.166119:0:22494:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f9952395230, osc lock ffff8f996686e130, flags 0x0 00000008:00010000:0.0:1775764854.166122:0:22494:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764854.166124:0:22494:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f9952395270, osc lock ffff8f996686e850, flags 0x0 00000008:00010000:0.0:1775764854.166127:0:22494:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764854.166129:0:22494:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99523952b0, osc lock ffff8f996686f0a0, flags 0x0 00010000:00010000:0.0:1775764854.166144:0:22494:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764854.166153:0:22494:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764854.166163:0:22494:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 6/0,2 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764854.166169:0:22494:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 6/0,2 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764854.166267:0:22494:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764854.166282:0:22494:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764854.166291:0:22494:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (1048576 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764854.166317:0:22494:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 6/0,2 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764854.166324:0:22494:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 6/0,2 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764854.166329:0:22494:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764854.166338:0:22494:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764854.166346:0:22494:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (2097152 3145727) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764854.166352:0:22494:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 6/0,2 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764854.166358:0:22494:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 6/0,2 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764854.166364:0:22494:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:1.0:1775764854.166618:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967052544:192.168.202.146@tcp:10:truncate.0 00000100:00100000:1.0:1775764854.166711:0:2380:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1775764854.166718:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967052672:192.168.202.146@tcp:10:truncate.0 00000100:00100000:0.0:1775764854.174941:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967052544:192.168.202.146@tcp:10:truncate.0 00000100:00100000:1.0:1775764854.176979:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967052672:192.168.202.146@tcp:10:truncate.0 00000080:00200000:2.0:1775764854.177067:0:22494:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0xee0:0x0] updating i_size 1048577 i_blocks 3 00000020:00010000:2.0:1775764854.177074:0:22494:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99713e9f38 (ffff8f9966dc22d8/0) at cl_lock_release():245 00000020:00010000:2.0:1775764854.177077:0:22494:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99713e9f38 (ffff8f9966dc22d8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764854.177081:0:22494:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9952395230 (ffff8f9966dc2248/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764854.177090:0:22494:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764854.177106:0:22494:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00010000:2.0:1775764854.177124:0:22494:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9952395270 (ffff8f9966dc2488/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764854.177129:0:22494:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 4/0,1 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764854.177137:0:22494:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00010000:2.0:1775764854.177146:0:22494:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99523952b0 (ffff8f9966dc2c68/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764854.177151:0:22494:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764854.177159:0:22494:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00010000:2.0:1775764854.177167:0:22494:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99713e9f38 (ffff8f9966dc22d8/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764854.177183:0:22494:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9952395230 (ffff8f9966dc22d8/1) at cl_lock_fini():72 00000020:00010000:2.0:1775764854.177189:0:22494:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9952395270 (ffff8f9966dc22d8/1) at cl_lock_fini():72 00000020:00010000:2.0:1775764854.177192:0:22494:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99523952b0 (ffff8f9966dc22d8/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764854.177244:0:22494:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xee0:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1775764854.177265:0:22494:0:(llite_lib.c:2524:ll_setattr_raw()) COMPLETED file f11a.sanity-pfl:[0x200000401:0xee0:0x0](ffff8f9952eb7448) current size 1048577, valid attrs 168, mode ffff, uid -1734053588, gid -1, new size 1048577, atime 1775764854.000000000, mtime 1775764854.000000000, ctime 1775764854.000000000, rc 0 00000080:00200000:1.0:1775764854.195326:0:22496:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764854.195352:0:22496:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764854.195368:0:22496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 21 lru_type: 1 00010000:00010000:1.0:1775764854.195384:0:22496:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 21 lru_type: 1 00000080:00200000:1.0:1775764854.195392:0:22496:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764854.195400:0:22496:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764854.195403:0:22496:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:1.0:1775764854.195443:0:22496:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac19c7bf8 00010000:00010000:1.0:1775764854.195447:0:22496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 21 lru_type: 1 00010000:00010000:1.0:1775764854.195455:0:22496:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00000080:00200000:1.0:1775764854.195470:0:22496:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764854.195477:0:22496:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d11a.sanity-pfl, flags=27 00000080:00200000:1.0:1775764854.195483:0:22496:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764854.195496:0:22496:0:(namei.c:1399:ll_atomic_open()) VFS Op:name=f11a.sanity-pfl, dir=[0x200000401:0xedf:0x0](ffff8f9952eedb88), file ffff8f9943735980, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1775764854.195501:0:22496:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=f11a.sanity-pfl, dir=[0x200000401:0xedf:0x0](ffff8f9952eedb88), intent=open 00000002:00010000:1.0:1775764854.195515:0:22496:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f11a.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000401:0xedf:0x0], intent: open flags 0304001 00010000:00010000:1.0:1775764854.195568:0:22496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c9de lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 22496 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.195575:0:22496:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c9de lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 22496 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764854.195586:0:22496:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764854.195597:0:22496:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e6d80 x1862022967052928/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1080/132280 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764854.195606:0:22496:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c9de lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 22496 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764854.195626:0:22496:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22496:1862022967052928:192.168.202.146@tcp:101:lfs.0 00000100:00100000:1.0:1775764854.198883:0:22496:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560240 going to sleep for 11 seconds 00000100:00100000:1.0:1775764854.205248:0:22496:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22496:1862022967052928:192.168.202.146@tcp:101:lfs.0 00010000:00010000:1.0:1775764854.205299:0:22496:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c9de lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 22496 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.205311:0:22496:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c9de lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 22496 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.205318:0:22496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c9de lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 22496 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.205325:0:22496:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c9de lrc: 3/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 22496 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.205332:0:22496:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c9de lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 22496 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.205338:0:22496:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970640200 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:1.0:1775764854.205342:0:22496:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c9de lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 22496 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.205348:0:22496:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c9de lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 22496 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.205355:0:22496:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c9de lrc: 5/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 22496 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.205361:0:22496:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1775764854.205366:0:22496:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c9de lrc: 3/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 22496 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.205372:0:22496:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c9de lrc: 0/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 22496 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764854.205408:0:22496:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff8f994f2e6d80 x1862022967052928/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1080/608 e 0 to 0 dl 1775764870 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764854.205421:0:22496:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0xedf:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1775764854.205685:0:22496:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764854.205703:0:22496:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764854.205718:0:22496:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00010000:00010000:1.0:1775764854.205730:0:22496:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00000080:00200000:1.0:1775764854.205803:0:22496:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764854.205810:0:22496:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764854.205814:0:22496:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:1.0:1775764854.205820:0:22496:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac19c7bf8 00010000:00010000:1.0:1775764854.205824:0:22496:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00010000:00010000:1.0:1775764854.205832:0:22496:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00000080:00200000:1.0:1775764854.205845:0:22496:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764854.205864:0:22496:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d11a.sanity-pfl, flags=267 00000080:00200000:1.0:1775764854.205871:0:22496:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88), inode mode 41ed mask 44 00000080:00200008:1.0:1775764854.205878:0:22496:0:(file.c:1057:ll_file_open()) START file d11a.sanity-pfl:[0x200000401:0xedf:0x0](ffff8f9952eedb88), flags 304000 00000080:00200080:1.0:1775764854.205916:0:22496:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88) cmd=c0086915 arg=19333d0 00000100:00100000:1.0:1775764854.205986:0:22496:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22496:1862022967053056:192.168.202.146@tcp:34:lfs.0 00000100:00100000:1.0:1775764854.206101:0:22496:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560240 going to sleep for 11 seconds 00000100:00100000:1.0:1775764854.212004:0:22496:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22496:1862022967053056:192.168.202.146@tcp:34:lfs.0 00000080:00200008:1.0:1775764854.212213:0:22496:0:(file.c:447:ll_file_release()) START file d11a.sanity-pfl:[0x200000401:0xedf:0x0](ffff8f9952eedb88), flags 304000 00010000:00010000:1.0:1775764854.212241:0:22496:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640600/0x36f9bdc9e421c967 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc220 expref: -99 pid: 22475 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:2.0:1775764854.223589:0:22499:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:2.0:1775764854.223617:0:22499:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764854.223636:0:22499:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00010000:00010000:2.0:1775764854.223652:0:22499:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00000080:00200000:2.0:1775764854.223660:0:22499:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764854.223667:0:22499:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:2.0:1775764854.223670:0:22499:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:2.0:1775764854.223677:0:22499:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac1d7bbf8 00010000:00010000:2.0:1775764854.223681:0:22499:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00010000:00010000:2.0:1775764854.223693:0:22499:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00000080:00200000:2.0:1775764854.223708:0:22499:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:2.0:1775764854.223716:0:22499:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d11a.sanity-pfl, flags=27 00000080:00200000:2.0:1775764854.223722:0:22499:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88), inode mode 41ed mask 1 00000080:00200000:2.0:1775764854.223735:0:22499:0:(namei.c:1399:ll_atomic_open()) VFS Op:name=f11a.sanity-pfl, dir=[0x200000401:0xedf:0x0](ffff8f9952eedb88), file ffff8f99613aa300, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1775764854.223839:0:22499:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=f11a.sanity-pfl, dir=[0x200000401:0xedf:0x0](ffff8f9952eedb88), intent=open 00000002:00010000:2.0:1775764854.223862:0:22499:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f11a.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000401:0xedf:0x0], intent: open flags 0304001 00010000:00010000:2.0:1775764854.223921:0:22499:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c9e5 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 22499 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764854.223929:0:22499:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c9e5 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 22499 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764854.223940:0:22499:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764854.223953:0:22499:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f99703d3100 x1862022967053312/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1080/132280 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764854.223962:0:22499:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c9e5 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 22499 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764854.223984:0:22499:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22499:1862022967053312:192.168.202.146@tcp:101:lfs.0 00000100:00100000:2.0:1775764854.224159:0:22499:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952af3180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764854.226651:0:22499:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22499:1862022967053312:192.168.202.146@tcp:101:lfs.0 00010000:00010000:2.0:1775764854.226674:0:22499:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c9e5 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 22499 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764854.226692:0:22499:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c9e5 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 22499 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764854.226699:0:22499:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c9e5 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 22499 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764854.226705:0:22499:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c9e5 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 22499 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764854.226713:0:22499:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c9e5 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 22499 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764854.226719:0:22499:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99706ea600 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:2.0:1775764854.226723:0:22499:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c9e5 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 22499 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764854.226731:0:22499:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c9e5 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 22499 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764854.226860:0:22499:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c9e5 lrc: 5/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 22499 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764854.226872:0:22499:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1775764854.226879:0:22499:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c9e5 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 22499 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764854.226885:0:22499:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421c9e5 lrc: 0/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 22499 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764854.226902:0:22499:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff8f99703d3100 x1862022967053312/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1080/608 e 0 to 0 dl 1775764870 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764854.226915:0:22499:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0xedf:0x0] suppgids 0 -1: rc -20 00000080:00200000:2.0:1775764854.227021:0:22499:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:2.0:1775764854.227272:0:22499:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764854.227284:0:22499:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00010000:00010000:2.0:1775764854.227293:0:22499:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00000080:00200000:2.0:1775764854.227298:0:22499:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764854.227303:0:22499:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:2.0:1775764854.227305:0:22499:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:2.0:1775764854.227309:0:22499:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac1d7bbf8 00010000:00010000:2.0:1775764854.227311:0:22499:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00010000:00010000:2.0:1775764854.227316:0:22499:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00000080:00200000:2.0:1775764854.227325:0:22499:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:2.0:1775764854.227330:0:22499:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d11a.sanity-pfl, flags=267 00000080:00200000:2.0:1775764854.227336:0:22499:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88), inode mode 41ed mask 44 00000080:00200008:2.0:1775764854.227342:0:22499:0:(file.c:1057:ll_file_open()) START file d11a.sanity-pfl:[0x200000401:0xedf:0x0](ffff8f9952eedb88), flags 304000 00000080:00200080:2.0:1775764854.227374:0:22499:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88) cmd=c0086915 arg=c083d0 00000100:00100000:2.0:1775764854.227419:0:22499:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22499:1862022967053440:192.168.202.146@tcp:34:lfs.0 00000100:00100000:2.0:1775764854.227494:0:22499:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952af3180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764854.229914:0:22499:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22499:1862022967053440:192.168.202.146@tcp:34:lfs.0 00000080:00200008:2.0:1775764854.230081:0:22499:0:(file.c:447:ll_file_release()) START file d11a.sanity-pfl:[0x200000401:0xedf:0x0](ffff8f9952eedb88), flags 304000 00010000:00010000:2.0:1775764854.230104:0:22499:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640600/0x36f9bdc9e421c967 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc220 expref: -99 pid: 22475 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:1.0:1775764854.244999:0:22502:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764854.245032:0:22502:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764854.245051:0:22502:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00010000:00010000:1.0:1775764854.245066:0:22502:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00000080:00200000:1.0:1775764854.245073:0:22502:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764854.245080:0:22502:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764854.245082:0:22502:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:1.0:1775764854.245088:0:22502:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac1d7bbf8 00010000:00010000:1.0:1775764854.245091:0:22502:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00010000:00010000:1.0:1775764854.245097:0:22502:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00000080:00200000:1.0:1775764854.245109:0:22502:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764854.245115:0:22502:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d11a.sanity-pfl, flags=27 00000080:00200000:1.0:1775764854.245135:0:22502:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764854.245144:0:22502:0:(namei.c:1399:ll_atomic_open()) VFS Op:name=f11a.sanity-pfl, dir=[0x200000401:0xedf:0x0](ffff8f9952eedb88), file ffff8f99427e3e00, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1775764854.245149:0:22502:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=f11a.sanity-pfl, dir=[0x200000401:0xedf:0x0](ffff8f9952eedb88), intent=open 00000002:00010000:1.0:1775764854.245160:0:22502:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f11a.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000401:0xedf:0x0], intent: open flags 0304001 00010000:00010000:1.0:1775764854.245277:0:22502:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c9ec lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 22502 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.245283:0:22502:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c9ec lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 22502 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764854.245292:0:22502:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764854.245301:0:22502:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e6d80 x1862022967053696/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1080/132280 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764854.245309:0:22502:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c9ec lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 22502 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764854.245326:0:22502:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22502:1862022967053696:192.168.202.146@tcp:101:lfs.0 00000100:00100000:1.0:1775764854.245480:0:22502:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560240 going to sleep for 11 seconds 00000100:00100000:1.0:1775764854.249239:0:22502:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22502:1862022967053696:192.168.202.146@tcp:101:lfs.0 00010000:00010000:1.0:1775764854.249260:0:22502:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c9ec lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 22502 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.249271:0:22502:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c9ec lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 22502 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.249277:0:22502:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c9ec lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 22502 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.249284:0:22502:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c9ec lrc: 3/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 22502 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.249310:0:22502:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c9ec lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 22502 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.249316:0:22502:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970640200 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:1.0:1775764854.249320:0:22502:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c9ec lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 22502 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.249326:0:22502:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c9ec lrc: 4/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 22502 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.249333:0:22502:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c9ec lrc: 5/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 22502 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.249340:0:22502:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1775764854.249345:0:22502:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c9ec lrc: 3/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 22502 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.249351:0:22502:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640200/0x36f9bdc9e421c9ec lrc: 0/0,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 22502 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764854.249366:0:22502:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff8f994f2e6d80 x1862022967053696/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1080/608 e 0 to 0 dl 1775764870 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764854.249379:0:22502:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0xedf:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1775764854.249504:0:22502:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764854.249521:0:22502:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764854.249534:0:22502:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00010000:00010000:1.0:1775764854.249545:0:22502:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00000080:00200000:1.0:1775764854.249567:0:22502:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764854.249572:0:22502:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764854.249574:0:22502:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:1.0:1775764854.249579:0:22502:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac1d7bbf8 00010000:00010000:1.0:1775764854.249583:0:22502:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00010000:00010000:1.0:1775764854.249590:0:22502:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00000080:00200000:1.0:1775764854.249603:0:22502:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764854.249610:0:22502:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d11a.sanity-pfl, flags=267 00000080:00200000:1.0:1775764854.249628:0:22502:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88), inode mode 41ed mask 44 00000080:00200008:1.0:1775764854.249636:0:22502:0:(file.c:1057:ll_file_open()) START file d11a.sanity-pfl:[0x200000401:0xedf:0x0](ffff8f9952eedb88), flags 304000 00000080:00200080:1.0:1775764854.249671:0:22502:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88) cmd=c0086915 arg=bed3d0 00000100:00100000:1.0:1775764854.249720:0:22502:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22502:1862022967053824:192.168.202.146@tcp:34:lfs.0 00000100:00100000:1.0:1775764854.249816:0:22502:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560240 going to sleep for 11 seconds 00000100:00100000:1.0:1775764854.253864:0:22502:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22502:1862022967053824:192.168.202.146@tcp:34:lfs.0 00000080:00200008:1.0:1775764854.255822:0:22502:0:(file.c:447:ll_file_release()) START file d11a.sanity-pfl:[0x200000401:0xedf:0x0](ffff8f9952eedb88), flags 304000 00010000:00010000:1.0:1775764854.255870:0:22502:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640600/0x36f9bdc9e421c967 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc220 expref: -99 pid: 22475 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:0.0:1775764854.267101:0:22504:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:0.0:1775764854.267134:0:22504:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764854.267156:0:22504:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00010000:00010000:0.0:1775764854.267260:0:22504:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00000080:00200000:0.0:1775764854.267270:0:22504:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764854.267278:0:22504:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:0.0:1775764854.267281:0:22504:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:0.0:1775764854.267289:0:22504:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac1c3bbf8 00010000:00010000:0.0:1775764854.267293:0:22504:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00010000:00010000:0.0:1775764854.267302:0:22504:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 28 lru_type: 1 00000080:00200000:0.0:1775764854.267319:0:22504:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:0.0:1775764854.267327:0:22504:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d11a.sanity-pfl, flags=25 00000080:00200000:0.0:1775764854.267334:0:22504:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764854.267345:0:22504:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88), inode mode 41ed mask 3 00000080:00200000:0.0:1775764854.267349:0:22504:0:(namei.c:1399:ll_atomic_open()) VFS Op:name=f11a.sanity-pfl, dir=[0x200000401:0xedf:0x0](ffff8f9952eedb88), file ffff8f994311aa80, open_flags 8042, mode 81b6 opened 0 00000080:00200000:0.0:1775764854.267357:0:22504:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=f11a.sanity-pfl, dir=[0x200000401:0xedf:0x0](ffff8f9952eedb88), intent=open|creat 00000002:00010000:0.0:1775764854.267395:0:22504:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f11a.sanity-pfl,[0x200000401:0xee1:0x0]) in obj [0x200000401:0xedf:0x0], intent: open|creat flags 0100103 00010000:00010000:0.0:1775764854.267449:0:22504:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c9f3 lrc: 3/0,1 mode: --/CW res: [0x200000401:0xedf:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 22504 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.267456:0:22504:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c9f3 lrc: 3/0,1 mode: --/CW res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 22504 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1775764854.267467:0:22504:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1775764854.267479:0:22504:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970a76300 x1862022967054080/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1080/66744 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1775764854.267489:0:22504:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c9f3 lrc: 3/0,1 mode: --/CW res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 22504 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764854.267510:0:22504:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job dd:3f83546c-01a9-4156-8a97-44c446245d3e:22504:1862022967054080:192.168.202.146@tcp:101:dd.0 00000100:00100000:0.0:1775764854.267796:0:22504:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102480 going to sleep for 11 seconds 00000100:00100000:2.0:1775764854.272858:0:22504:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job dd:3f83546c-01a9-4156-8a97-44c446245d3e:22504:1862022967054080:192.168.202.146@tcp:101:dd.0 00010000:00010000:2.0:1775764854.272886:0:22504:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c9f3 lrc: 4/0,1 mode: --/CW res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc457 expref: -99 pid: 22504 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764854.272901:0:22504:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c9f3 lrc: 4/0,1 mode: --/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc457 expref: -99 pid: 22504 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764854.272909:0:22504:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c9f3 lrc: 4/0,1 mode: --/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc457 expref: -99 pid: 22504 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764854.272920:0:22504:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c9f3 lrc: 4/0,1 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc457 expref: -99 pid: 22504 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764854.272929:0:22504:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c9f3 lrc: 4/0,1 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc457 expref: -99 pid: 22504 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764854.272942:0:22504:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c9f3 lrc: 5/1,1 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc457 expref: -99 pid: 22504 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764854.272949:0:22504:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c9f3 lrc: 5/1,1 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc457 expref: -99 pid: 22504 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764854.272956:0:22504:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c9f3 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc457 expref: -99 pid: 22504 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764854.273007:0:22504:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=3 disposition=2b, status=0 req@ffff8f9970a76300 x1862022967054080/t8589934628(8589934628) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1080/1160 e 0 to 0 dl 1775764870 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764854.273020:0:22504:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f9970a76300 x1862022967054080/t8589934628(8589934628) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1080/1160 e 0 to 0 dl 1775764870 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764854.273035:0:22504:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c9f3 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc457 expref: -99 pid: 22504 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764854.273049:0:22504:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c9f3 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc457 expref: -99 pid: 22504 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764854.273060:0:22504:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/3808 (0 0) 00000080:00200000:2.0:1775764854.273067:0:22504:0:(namei.c:943:ll_intent_lock()) intent lock 3 on i1 [0x200000401:0xedf:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764854.273157:0:22504:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0xee0:0x0](ffff8f9952eb7448): rc = 0 00000080:00200000:2.0:1775764854.273263:0:22504:0:(file.c:6766:ll_layout_conf()) [0x200000401:0xee0:0x0]: layout version change: 7 -> 7 00000080:00010000:2.0:1775764854.273278:0:22504:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xee0:0x0](ffff8f9952eb7448) for lock 0x36f9bdc9e421c9f3 00000080:00010000:2.0:1775764854.273291:0:22504:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xee0:0x0](ffff8f9952eb7448) 00000080:00200008:2.0:1775764854.273307:0:22504:0:(file.c:1057:ll_file_open()) START file f11a.sanity-pfl:[0x200000401:0xee0:0x0](ffff8f9952eb7448), flags 100102 00000002:00100000:2.0:1775764854.273330:0:22504:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f9970a76300 x1862022967054080/t8589934628(8589934628) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1080/1160 e 0 to 0 dl 1775764870 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1775764854.273373:0:22504:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c9f3 from it ffff8f9970ad0d20 00010000:00010000:2.0:1775764854.273378:0:22504:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c9f3 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fc457 expref: -99 pid: 22504 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764854.273387:0:22504:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c9f3 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fc457 expref: -99 pid: 22504 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764854.273410:0:22504:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xee0:0x0](ffff8f9952eb7448), inode mode 81a4 mask 46 00000080:00200008:2.0:1775764854.273453:0:22504:0:(llite_lib.c:2274:ll_setattr_raw()) START file f11a.sanity-pfl:[0x200000401:0xee0:0x0](ffff8f9952eb7448) current size 1048577, valid attrs 2068, mode 0, uid 514, gid 0, new size 2097152, atime 1775764854.000000000, mtime 1775764854.000000000, ctime 1775764854.000000000 00010000:00010000:2.0:1775764854.273475:0:22504:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c9f3 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5fc457 expref: -99 pid: 22504 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764854.273493:0:22504:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c9d7 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fc3bd expref: -99 pid: 22494 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764854.273530:0:22504:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c9f3 lrc: 1/0,0 mode: --/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fc457 expref: -99 pid: 22504 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764854.273537:0:22504:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764854.273540:0:22504:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cc200/0x36f9bdc9e421c9f3 lrc: 0/0,0 mode: --/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fc457 expref: -99 pid: 22504 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000020:00100000:2.0:1775764854.273557:0:22504:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1775764854.273563:0:22504:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f99703d3100 x1862022967054336/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764854.273585:0:22504:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job dd:3f83546c-01a9-4156-8a97-44c446245d3e:22504:1862022967054336:192.168.202.146@tcp:36:dd.0 00000100:00100000:2.0:1775764854.273898:0:22504:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952af3180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764854.277133:0:22504:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job dd:3f83546c-01a9-4156-8a97-44c446245d3e:22504:1862022967054336:192.168.202.146@tcp:36:dd.0 00000080:00200000:2.0:1775764854.277206:0:22504:0:(file.c:1815:ll_io_set_mirror()) f11a.sanity-pfl: desiginated mirror: 0 00000080:00200000:2.0:1775764854.277213:0:22504:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xee0:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764854.277231:0:22504:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x10 00020000:00200000:2.0:1775764854.277256:0:22504:0:(lov_io.c:860:lov_io_iter_init()) component[3] flags 0x0 00020000:00200000:2.0:1775764854.277258:0:22504:0:(lov_io.c:860:lov_io_iter_init()) component[4] flags 0x0 00000080:00200000:2.0:1775764854.277261:0:22504:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:2.0:1775764854.277303:0:22504:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764854.277308:0:22504:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f9960127930, osc lock ffff8f994f1a2ab0, flags 0x0 00000008:00010000:2.0:1775764854.277331:0:22504:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764854.277333:0:22504:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f9960127970, osc lock ffff8f994f1a38f0, flags 0x0 00000008:00010000:2.0:1775764854.277336:0:22504:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764854.277338:0:22504:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99601279b0, osc lock ffff8f994f1a3b50, flags 0x0 00010000:00010000:2.0:1775764854.277359:0:22504:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764854.277375:0:22504:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764854.277389:0:22504:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 6/0,2 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764854.277398:0:22504:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 6/0,2 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764854.277405:0:22504:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764854.277416:0:22504:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764854.277425:0:22504:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (1048576 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764854.277438:0:22504:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 6/0,2 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764854.277446:0:22504:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 6/0,2 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764854.277453:0:22504:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764854.277463:0:22504:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764854.277471:0:22504:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (2097152 3145727) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764854.277479:0:22504:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 6/0,2 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764854.277487:0:22504:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 6/0,2 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764854.277494:0:22504:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000100:00100000:3.0:1775764854.277594:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967054464:192.168.202.146@tcp:10:dd.0 00000100:00100000:3.0:1775764854.283432:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967054464:192.168.202.146@tcp:10:dd.0 00000080:00200000:2.0:1775764854.283503:0:22504:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0xee0:0x0] updating i_size 2097152 i_blocks 3 00000020:00010000:2.0:1775764854.283511:0:22504:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970942c68 (ffff8f994ef582d8/0) at cl_lock_release():245 00000020:00010000:2.0:1775764854.283514:0:22504:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970942c68 (ffff8f994ef582d8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764854.283519:0:22504:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9960127930 (ffff8f994ef58758/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764854.283529:0:22504:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764854.283544:0:22504:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00010000:2.0:1775764854.283559:0:22504:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9960127970 (ffff8f994ef58e18/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764854.283565:0:22504:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 4/0,1 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764854.283574:0:22504:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00010000:2.0:1775764854.283583:0:22504:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99601279b0 (ffff8f994ef58908/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764854.283587:0:22504:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764854.283595:0:22504:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00010000:2.0:1775764854.283603:0:22504:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970942c68 (ffff8f994ef582d8/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764854.283607:0:22504:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9960127930 (ffff8f994ef582d8/1) at cl_lock_fini():72 00000020:00010000:2.0:1775764854.283681:0:22504:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9960127970 (ffff8f994ef582d8/1) at cl_lock_fini():72 00000020:00010000:2.0:1775764854.283685:0:22504:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99601279b0 (ffff8f994ef582d8/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764854.283813:0:22504:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xee0:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1775764854.283835:0:22504:0:(llite_lib.c:2524:ll_setattr_raw()) COMPLETED file f11a.sanity-pfl:[0x200000401:0xee0:0x0](ffff8f9952eb7448) current size 2097152, valid attrs 2168, mode 0, uid 514, gid 0, new size 2097152, atime 1775764854.000000000, mtime 1775764854.000000000, ctime 1775764854.000000000, rc 0 00000080:00200080:2.0:1775764854.283900:0:22504:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200000401:0xee0:0x0](ffff8f9952eb7448) cmd=80306d02 arg=7ffda0b6e4d0 00020000:00000080:2.0:1775764854.283911:0:22504:0:(lov_obd.c:961:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: cmd=80306d02 len=0 karg=0000000000000000 uarg=00007ffda0b6e4d0 00000020:00000080:2.0:1775764854.283915:0:22504:0:(lov_obd.c:966:lov_iocontrol()) lustre-clilov-ffff8f99523ee800: iocontrol from 'dd' cmd=80306d02 _IOR('m', 2, 48) unknown: rc = -25 00000080:00200008:2.0:1775764854.283922:0:22504:0:(file.c:5170:ll_file_seek()) START file f11a.sanity-pfl:[0x200000401:0xee0:0x0], offset: 2097152, type: SEEK_CUR 00000080:00200008:2.0:1775764854.283931:0:22504:0:(file.c:5201:ll_file_seek()) COMPLETED file f11a.sanity-pfl:[0x200000401:0xee0:0x0], offset: 2097152, type: SEEK_CUR, rc = 2097152 00000080:00200008:2.0:1775764854.283992:0:22504:0:(file.c:2612:do_file_write_iter()) START file f11a.sanity-pfl:[0x200000401:0xee0:0x0], ppos: 2097152, count: 1024 00000080:00200000:2.0:1775764854.284006:0:22504:0:(rw26.c:663:ll_write_begin()) Writing 512 of 0 to 1024 bytes 00000080:00200000:2.0:1775764854.284011:0:22504:0:(file.c:2588:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:2.0:1775764854.284033:0:22504:0:(file.c:2058:ll_file_io_generic()) f11a.sanity-pfl: write ppos: 2097152, bytes: 1024 00000080:00200000:2.0:1775764854.284037:0:22504:0:(file.c:1815:ll_io_set_mirror()) f11a.sanity-pfl: desiginated mirror: 0 00000020:00200000:2.0:1775764854.284043:0:22504:0:(cl_io.c:214:cl_io_rw_init()) header@000000008dd138bb[0x110, 1, [0x200000401:0xee0:0x0] hash] 00000020:00200000:2.0:1775764854.284045:0:22504:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [2097152, 2098176) 0 0 00000080:00200000:2.0:1775764854.284048:0:22504:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xee0:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764854.284063:0:22504:0:(file.c:2136:ll_file_io_generic()) Range lock [512, 512] 00020000:00200000:2.0:1775764854.284070:0:22504:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 2097152 chunk: [2097152, 2098176) 2098176, 1024 00020000:00200000:2.0:1775764854.284074:0:22504:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x10 00000080:00200000:2.0:1775764854.284097:0:22504:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [512, 512] 00000008:00010000:2.0:1775764854.284111:0:22504:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764854.284115:0:22504:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994166ff30, osc lock ffff8f994f1a3b50, flags 0x0 00010000:00010000:2.0:1775764854.284137:0:22504:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764854.284149:0:22504:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (2097152 2101247) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764854.284160:0:22504:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 6/0,2 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764854.284167:0:22504:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 6/0,2 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764854.284174:0:22504:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764854.284185:0:22504:0:(vvp_io.c:1286:vvp_io_write_start()) f11a.sanity-pfl: write [2097152, 2098176) append 0 append lockpos 0 00000080:00200000:2.0:1775764854.284194:0:22504:0:(rw26.c:663:ll_write_begin()) Writing 512 of 0 to 1024 bytes 00000080:00200000:2.0:1775764854.284238:0:22504:0:(rw26.c:858:ll_write_end()) pos 2097152, len 1024, copied 1024 00000080:00200000:2.0:1775764854.284246:0:22504:0:(rw26.c:916:ll_write_end()) page@00000000fc89cfa4[3 00000000a87f46e8 2 1 00000000a75d5000] 00000080:00200000:2.0:1775764854.284249:0:22504:0:(rw26.c:916:ll_write_end()) vmpage @00000000ec79eeb1 17ffffc0001001 3:0 ffff8f9970675e00 512 lru 00000080:00200000:2.0:1775764854.284256:0:22504:0:(rw26.c:916:ll_write_end()) osc-page@000000009d7d1f40 512: 1< 0 - - > 2< 2097152 0 0 0x0 0x0 | 000000004b957d6e 000000000dd2e89b > 3< 0 0 > 4< 0 0 8 464650240 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.284258:0:22504:0:(rw26.c:916:ll_write_end()) end page@00000000fc89cfa4 00000080:00200000:2.0:1775764854.284259:0:22504:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1775764854.284264:0:22504:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:2.0:1775764854.284282:0:22504:0:(osc_cache.c:848:osc_extent_find()) ### extent: 000000007bd0fffd ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764854.284301:0:22504:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8f9952eb75f8, count 1, dirtied 1 00000080:00200000:2.0:1775764854.284305:0:22504:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:2.0:1775764854.284308:0:22504:0:(vvp_io.c:1353:vvp_io_write_start()) f11a.sanity-pfl: write bytes 0, result: 1024 00000020:00010000:2.0:1775764854.284321:0:22504:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970942c68 (ffff8f994ef582d8/0) at cl_lock_release():245 00000020:00010000:2.0:1775764854.284324:0:22504:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970942c68 (ffff8f994ef582d8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764854.284327:0:22504:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994166ff30 (ffff8f994ef58e18/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764854.284332:0:22504:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764854.284341:0:22504:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 4/0,0 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00010000:2.0:1775764854.284352:0:22504:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970942c68 (ffff8f994ef582d8/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764854.284354:0:22504:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994166ff30 (ffff8f994ef582d8/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764854.284360:0:22504:0:(vvp_io.c:517:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:2.0:1775764854.284364:0:22504:0:(file.c:2173:ll_file_io_generic()) Range unlock [512, 512] 00000080:00200000:2.0:1775764854.284379:0:22504:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xee0:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764854.284382:0:22504:0:(file.c:2206:ll_file_io_generic()) f11a.sanity-pfl: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:2.0:1775764854.284389:0:22504:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0xee0:0x0] updating i_size 2098176 i_blocks 3 00000080:00200000:2.0:1775764854.284392:0:22504:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:2.0:1775764854.284425:0:22504:0:(file.c:447:ll_file_release()) START file f11a.sanity-pfl:[0x200000401:0xee0:0x0](ffff8f9952eb7448), flags 100002 00000002:00100000:2.0:1775764854.284473:0:22504:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f9970a76300 x1862022967054080/t8589934628(8589934628) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1080/1160 e 0 to 0 dl 1775764870 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dd.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764854.284485:0:22504:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764854.284490:0:22504:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f99703d3b80 x1862022967054720/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/1048 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764854.284509:0:22504:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job dd:3f83546c-01a9-4156-8a97-44c446245d3e:22504:1862022967054720:192.168.202.146@tcp:35:dd.0 00000100:00100000:2.0:1775764854.284714:0:22504:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952af3180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764854.288367:0:22504:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job dd:3f83546c-01a9-4156-8a97-44c446245d3e:22504:1862022967054720:192.168.202.146@tcp:35:dd.0 00000002:00100000:2.0:1775764854.288400:0:22504:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f9970a76300 x1862022967054080/t8589934628(8589934628) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1080/1160 e 0 to 0 dl 1775764870 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dd.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1775764854.304728:0:22505:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764854.305016:0:22505:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764854.305037:0:22505:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 28 lru_type: 1 00010000:00010000:1.0:1775764854.305092:0:22505:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 28 lru_type: 1 00000080:00200000:1.0:1775764854.305099:0:22505:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764854.305107:0:22505:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764854.305111:0:22505:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:1.0:1775764854.305117:0:22505:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac1c3bb58 00010000:00010000:1.0:1775764854.305121:0:22505:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 28 lru_type: 1 00010000:00010000:1.0:1775764854.305128:0:22505:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00000080:00200000:1.0:1775764854.305143:0:22505:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764854.305151:0:22505:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d11a.sanity-pfl, flags=24 00000080:00200000:1.0:1775764854.305156:0:22505:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764854.305166:0:22505:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=f11a.sanity-pfl, dir=[0x200000401:0xedf:0x0](ffff8f9952eedb88), flags=8 00000080:00200000:1.0:1775764854.305250:0:22505:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=f11a.sanity-pfl, dir=[0x200000401:0xedf:0x0](ffff8f9952eedb88), intent=getattr 00000002:00010000:1.0:1775764854.305265:0:22505:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f11a.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000401:0xedf:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764854.305311:0:22505:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c9fa lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 22505 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.305319:0:22505:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c9fa lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 22505 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.305329:0:22505:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c9fa lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 22505 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764854.305374:0:22505:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ls:3f83546c-01a9-4156-8a97-44c446245d3e:22505:1862022967054848:192.168.202.146@tcp:101:ls.0 00000100:00100000:1.0:1775764854.305504:0:22505:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560240 going to sleep for 11 seconds 00000100:00100000:1.0:1775764854.309666:0:22505:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ls:3f83546c-01a9-4156-8a97-44c446245d3e:22505:1862022967054848:192.168.202.146@tcp:101:ls.0 00010000:00010000:1.0:1775764854.309692:0:22505:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c9fa lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc481 expref: -99 pid: 22505 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.309707:0:22505:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c9fa lrc: 4/1,0 mode: --/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc481 expref: -99 pid: 22505 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.309714:0:22505:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c9fa lrc: 4/1,0 mode: --/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc481 expref: -99 pid: 22505 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.309725:0:22505:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c9fa lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc481 expref: -99 pid: 22505 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.309734:0:22505:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c9fa lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc481 expref: -99 pid: 22505 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.309773:0:22505:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c9fa lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc481 expref: -99 pid: 22505 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.309780:0:22505:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c9fa lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc481 expref: -99 pid: 22505 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.309787:0:22505:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c9fa lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc481 expref: -99 pid: 22505 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764854.309821:0:22505:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994f2e6d80 x1862022967054848/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/1136 e 0 to 0 dl 1775764870 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'ls.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764854.309831:0:22505:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c9fa lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc481 expref: -99 pid: 22505 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1775764854.309844:0:22505:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c9fa lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc481 expref: -99 pid: 22505 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.309863:0:22505:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/3808 (0 0) 00000080:00200000:1.0:1775764854.309870:0:22505:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0xedf:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1775764854.309963:0:22505:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0xee0:0x0](ffff8f9952eb7448): rc = 0 00000080:00200000:1.0:1775764854.309983:0:22505:0:(file.c:6766:ll_layout_conf()) [0x200000401:0xee0:0x0]: layout version change: 7 -> 7 00000080:00010000:1.0:1775764854.309993:0:22505:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xee0:0x0](ffff8f9952eb7448) for lock 0x36f9bdc9e421c9fa 00000080:00010000:1.0:1775764854.310003:0:22505:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xee0:0x0](ffff8f9952eb7448) 00000080:00010000:1.0:1775764854.310006:0:22505:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c9fa from it ffffafbac1c3bb20 00010000:00010000:1.0:1775764854.310009:0:22505:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c9fa lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fc481 expref: -99 pid: 22505 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.310017:0:22505:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c9fa lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fc481 expref: -99 pid: 22505 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200008:1.0:1775764854.310051:0:22505:0:(file.c:6226:ll_getattr_dentry()) START file f11a.sanity-pfl:[0x200000401:0xee0:0x0](ffff8f9952eb7448), request_mask 606, flags 0, foreign 0 00000080:00200000:1.0:1775764854.310057:0:22505:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xee0:0x0](ffff8f9952eb7448),name=f11a.sanity-pfl 00000002:00010000:1.0:1775764854.310074:0:22505:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xee0:0x0]) in obj [0x200000401:0xedf:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764854.310081:0:22505:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c9fa lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fc481 expref: -99 pid: 22505 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764854.310105:0:22505:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c9fa lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fc481 expref: -99 pid: 22505 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764854.310111:0:22505:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0xedf:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764854.310116:0:22505:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xee0:0x0](ffff8f9952eb7448) 00000080:00010000:1.0:1775764854.310119:0:22505:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xee0:0x0](ffff8f9952eb7448) for lock 0x36f9bdc9e421c9fa 00000080:00010000:1.0:1775764854.310122:0:22505:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c9fa from it ffffafbac1c3bcb8 00010000:00010000:1.0:1775764854.310124:0:22505:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c9fa lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fc481 expref: -99 pid: 22505 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764854.310130:0:22505:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c9fa lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fc481 expref: -99 pid: 22505 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764854.310152:0:22505:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xee0:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764854.310161:0:22505:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xee0:0x0] 00000008:00010000:1.0:1775764854.310200:0:22505:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764854.310203:0:22505:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994304e930, osc lock ffff8f995875d690, flags 0x400001002 00000008:00010000:1.0:1775764854.310218:0:22505:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764854.310219:0:22505:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994304e970, osc lock ffff8f995875c390, flags 0x400001002 00000008:00010000:1.0:1775764854.310234:0:22505:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764854.310236:0:22505:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994304e9b0, osc lock ffff8f995875ddb0, flags 0x400001002 00010000:00010000:1.0:1775764854.310251:0:22505:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764854.310262:0:22505:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1775764854.310289:0:22505:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764854.310292:0:22505:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764854.310299:0:22505:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 4/0,0 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764854.310315:0:22505:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764854.310323:0:22505:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (1048576 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1775764854.310330:0:22505:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764854.310332:0:22505:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764854.310338:0:22505:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 4/0,0 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764854.310350:0:22505:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764854.310357:0:22505:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (2097152 3145727) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1775764854.310381:0:22505:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764854.310384:0:22505:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 6/0,1 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764854.310390:0:22505:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 5/0,0 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:1.0:1775764854.310398:0:22505:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764854.310408:0:22505:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0xee0:0x0] updating i_size 2098176 i_blocks 3 00000020:00010000:1.0:1775764854.310412:0:22505:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705f8ff8 (ffff8f99705b8878/0) at cl_lock_release():245 00000020:00010000:1.0:1775764854.310415:0:22505:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705f8ff8 (ffff8f99705b8878/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764854.310417:0:22505:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994304e930 (ffff8f99705b83f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764854.310421:0:22505:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994304e970 (ffff8f99705b8e18/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764854.310423:0:22505:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994304e9b0 (ffff8f99705b86c8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764854.310425:0:22505:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705f8ff8 (ffff8f99705b8878/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764854.310428:0:22505:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994304e930 (ffff8f99705b8878/1) at cl_lock_fini():72 00000020:00010000:1.0:1775764854.310432:0:22505:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994304e970 (ffff8f99705b8878/1) at cl_lock_fini():72 00000020:00010000:1.0:1775764854.310435:0:22505:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994304e9b0 (ffff8f99705b8878/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764854.310468:0:22505:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xee0:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764854.310517:0:22505:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764854.310526:0:22505:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764854.310534:0:22505:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00010000:00010000:1.0:1775764854.310541:0:22505:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00000080:00200000:1.0:1775764854.310561:0:22505:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764854.310565:0:22505:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764854.310567:0:22505:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:1.0:1775764854.310571:0:22505:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac1c3bbf8 00010000:00010000:1.0:1775764854.310574:0:22505:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00010000:00010000:1.0:1775764854.310581:0:22505:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 30 lru_type: 1 00000080:00200000:1.0:1775764854.310589:0:22505:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764854.310596:0:22505:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d11a.sanity-pfl, flags=24 00000080:00200000:1.0:1775764854.310601:0:22505:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764854.310604:0:22505:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f11a.sanity-pfl, flags=8 00000080:00200000:1.0:1775764854.310626:0:22505:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764854.310633:0:22505:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764854.310639:0:22505:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 30 lru_type: 1 00010000:00010000:1.0:1775764854.310646:0:22505:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 30 lru_type: 1 00000080:00200000:1.0:1775764854.310652:0:22505:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764854.310656:0:22505:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764854.310659:0:22505:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:1.0:1775764854.310662:0:22505:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac1c3bbf8 00010000:00010000:1.0:1775764854.310665:0:22505:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 30 lru_type: 1 00010000:00010000:1.0:1775764854.310685:0:22505:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00000080:00200000:1.0:1775764854.310694:0:22505:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764854.310697:0:22505:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d11a.sanity-pfl, flags=25 00000080:00200000:1.0:1775764854.310701:0:22505:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764854.310704:0:22505:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f11a.sanity-pfl, flags=9 00000080:00200000:1.0:1775764854.332556:0:22506:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764854.332585:0:22506:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764854.332605:0:22506:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00010000:00010000:1.0:1775764854.332713:0:22506:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00000080:00200000:1.0:1775764854.332721:0:22506:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764854.332728:0:22506:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764854.332732:0:22506:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:1.0:1775764854.333131:0:22506:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac1df3bf8 00010000:00010000:1.0:1775764854.333137:0:22506:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00010000:00010000:1.0:1775764854.333147:0:22506:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 32 lru_type: 1 00000080:00200000:1.0:1775764854.333161:0:22506:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764854.333259:0:22506:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d11a.sanity-pfl, flags=25 00000080:00200000:1.0:1775764854.333543:0:22506:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764854.333553:0:22506:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f11a.sanity-pfl, flags=777 00000080:00200000:1.0:1775764854.333558:0:22506:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xee0:0x0](ffff8f9952eb7448), inode mode 81a4 mask 52 00000080:00200008:1.0:1775764854.333566:0:22506:0:(file.c:1057:ll_file_open()) START file f11a.sanity-pfl:[0x200000401:0xee0:0x0](ffff8f9952eb7448), flags 102101 00000002:00010000:1.0:1775764854.333582:0:22506:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xee0:0x0]) in obj [0x200000401:0xedf:0x0], intent: open|creat flags 04240102102 00010000:00010000:1.0:1775764854.333804:0:22506:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421ca01 lrc: 3/0,1 mode: --/CW res: [0x200000401:0xedf:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 22506 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.333814:0:22506:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421ca01 lrc: 3/0,1 mode: --/CW res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 22506 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764854.333828:0:22506:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764854.333840:0:22506:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e6d80 x1862022967055104/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1072/66744 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764854.333861:0:22506:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421ca01 lrc: 3/0,1 mode: --/CW res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 22506 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764854.333884:0:22506:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job rwv:3f83546c-01a9-4156-8a97-44c446245d3e:22506:1862022967055104:192.168.202.146@tcp:101:rwv.0 00000100:00100000:1.0:1775764854.334078:0:22506:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560240 going to sleep for 11 seconds 00000100:00100000:0.0:1775764854.337038:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764854.337125:0:5561:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978417280 00000100:00100000:1.0:1775764854.337137:0:5561:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978417280 00000100:00100000:1.0:1775764854.337204:0:5561:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24227 00000100:00100000:1.0:1775764854.337214:0:5561:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970a74700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+5:17506:x1862022978417280:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764854.337232:0:5561:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c9fa lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x2aac6a50e5fc481 expref: -99 pid: 22505 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764854.337302:0:5561:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764854.337308:0:5561:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970a74700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:3f83546c-01a9-4156-8a97-44c446245d3e+5:17506:x1862022978417280:12345-192.168.202.146@tcp:104: Request processed in 97us (282us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764854.337345:0:5561:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24227 00010000:00010000:2.0:1775764854.337814:0:5572:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764854.337821:0:5572:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c9fa lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x2aac6a50e5fc481 expref: -99 pid: 22505 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764854.337830:0:5572:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970640c00 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:2.0:1775764854.337834:0:5572:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c9fa lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x2aac6a50e5fc481 expref: -99 pid: 22505 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764854.337842:0:5572:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c9fa lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x2aac6a50e5fc481 expref: -99 pid: 22505 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764854.337861:0:5572:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c9fa lrc: 3/0,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x2aac6a50e5fc481 expref: -99 pid: 22505 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764854.337889:0:5572:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c9d7 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fc3bd expref: -99 pid: 22494 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764854.337927:0:5572:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c9fa lrc: 2/0,0 mode: --/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x2aac6a50e5fc481 expref: -99 pid: 22505 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764854.337933:0:5572:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764854.337964:0:5572:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c9fa lrc: 1/0,0 mode: --/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x2aac6a50e5fc481 expref: -99 pid: 22505 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764854.337970:0:5572:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640c00/0x36f9bdc9e421c9fa lrc: 0/0,0 mode: --/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x2aac6a50e5fc481 expref: -99 pid: 22505 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764854.337984:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764854.337985:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764854.337987:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764854.337994:0:2381:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1775764854.338008:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967055232:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1775764854.341971:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764854.342444:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967055232:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764854.343259:0:22506:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job rwv:3f83546c-01a9-4156-8a97-44c446245d3e:22506:1862022967055104:192.168.202.146@tcp:101:rwv.0 00010000:00010000:1.0:1775764854.343289:0:22506:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421ca01 lrc: 4/0,1 mode: --/CW res: [0x200000401:0xee0:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc48f expref: -99 pid: 22506 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.343301:0:22506:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421ca01 lrc: 4/0,1 mode: --/CW res: [0x200000401:0xee0:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc48f expref: -99 pid: 22506 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.343312:0:22506:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421ca01 lrc: 4/0,1 mode: CW/CW res: [0x200000401:0xee0:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc48f expref: -99 pid: 22506 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.343323:0:22506:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421ca01 lrc: 4/0,1 mode: CW/CW res: [0x200000401:0xee0:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc48f expref: -99 pid: 22506 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764854.343337:0:22506:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=3 disposition=200002b, status=0 req@ffff8f994f2e6d80 x1862022967055104/t8589934632(8589934632) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1775764870 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'rwv.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764854.343349:0:22506:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f994f2e6d80 x1862022967055104/t8589934632(8589934632) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1775764870 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'rwv.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764854.343362:0:22506:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421ca01 lrc: 3/0,1 mode: CW/CW res: [0x200000401:0xee0:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc48f expref: -99 pid: 22506 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.343372:0:22506:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 8 res 8589935617/3808 (0 0) 00000080:00200000:1.0:1775764854.343380:0:22506:0:(namei.c:943:ll_intent_lock()) intent lock 3 on i1 [0x200000401:0xedf:0x0] suppgids 0 0: rc 0 00000080:00010000:1.0:1775764854.343475:0:22506:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xee0:0x0](ffff8f9952eb7448) for lock 0x36f9bdc9e421ca01 00000080:00010000:1.0:1775764854.343482:0:22506:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421ca01 from it ffffafbac1df3bf0 00010000:00010000:1.0:1775764854.343527:0:22506:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421ca01 lrc: 3/0,1 mode: CW/CW res: [0x200000401:0xee0:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc48f expref: -99 pid: 22506 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.343536:0:22506:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421ca01 lrc: 2/0,0 mode: CW/CW res: [0x200000401:0xee0:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc48f expref: -99 pid: 22506 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000002:00100000:1.0:1775764854.343562:0:22506:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f994f2e6d80 x1862022967055104/t8589934632(8589934632) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1775764870 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'rwv.0' uid:0 gid:0 projid:0 00000080:00200008:1.0:1775764854.343667:0:22506:0:(file.c:2612:do_file_write_iter()) START file f11a.sanity-pfl:[0x200000401:0xee0:0x0], ppos: 0, count: 1047553 00000080:00200000:1.0:1775764854.343684:0:22506:0:(file.c:2058:ll_file_io_generic()) f11a.sanity-pfl: write ppos: 0, bytes: 1047553 00000080:00200000:1.0:1775764854.343690:0:22506:0:(file.c:1815:ll_io_set_mirror()) f11a.sanity-pfl: desiginated mirror: 0 00000080:00200000:1.0:1775764854.343700:0:22506:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0xee0:0x0] updating i_size 2098176 i_blocks 3 00000020:00200000:1.0:1775764854.343709:0:22506:0:(cl_io.c:214:cl_io_rw_init()) header@000000008dd138bb[0x110, 2, [0x200000401:0xee0:0x0] hash] 00000020:00200000:1.0:1775764854.343711:0:22506:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [2098176, 3145729) 0 1 00000080:00200000:1.0:1775764854.343714:0:22506:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xee0:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764854.343727:0:22506:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xee0:0x0] ignore/verify layout 0/0, layout version 7 need write layout 1, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764854.343731:0:22506:0:(vvp_io.c:355:vvp_io_fini()) [0x200000401:0xee0:0x0] write layout, type 2 [0x200400, 0x300001) 00000080:00010000:1.0:1775764854.343785:0:22506:0:(file.c:6988:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200000401:0xee0:0x0](ffff8f9952eb7448) 00000002:00010000:1.0:1775764854.343795:0:22506:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xee0:0x0]) in obj [0x200000401:0xee0:0x0], intent: layout flags 02 00010000:00010000:1.0:1775764854.343810:0:22506:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c9d7 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x2aac6a50e5fc3bd expref: -99 pid: 22494 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764854.343833:0:22506:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0xee0:0x0]: losing layout lock 00010000:00010000:1.0:1775764854.343884:0:22506:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c9d7 lrc: 1/0,0 mode: --/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fc3bd expref: -99 pid: 22494 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764854.343891:0:22506:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764854.343894:0:22506:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cdc00/0x36f9bdc9e421c9d7 lrc: 0/0,0 mode: --/CR res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x2aac6a50e5fc3bd expref: -99 pid: 22494 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764854.343933:0:22506:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(EX) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421ca08 lrc: 3/0,1 mode: --/EX res: [0x200000401:0xee0:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 22506 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.343939:0:22506:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421ca08 lrc: 3/0,1 mode: --/EX res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 22506 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764854.343949:0:22506:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764854.343955:0:22506:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e4000 x1862022967055488/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 376/896 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764854.343963:0:22506:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421ca08 lrc: 3/0,1 mode: --/EX res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 22506 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764854.343979:0:22506:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job rwv:3f83546c-01a9-4156-8a97-44c446245d3e:22506:1862022967055488:192.168.202.146@tcp:101:rwv.0 00000100:00100000:1.0:1775764854.344050:0:22506:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560240 going to sleep for 11 seconds 00000100:00100000:1.0:1775764854.348940:0:22506:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job rwv:3f83546c-01a9-4156-8a97-44c446245d3e:22506:1862022967055488:192.168.202.146@tcp:101:rwv.0 00010000:00010000:1.0:1775764854.348964:0:22506:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode CR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421ca08 lrc: 4/0,1 mode: --/EX res: [0x200000401:0xee0:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc4a4 expref: -99 pid: 22506 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.348977:0:22506:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421ca08 lrc: 4/0,1 mode: --/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc4a4 expref: -99 pid: 22506 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.348988:0:22506:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421ca08 lrc: 4/0,1 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc4a4 expref: -99 pid: 22506 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.348998:0:22506:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421ca08 lrc: 4/0,1 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc4a4 expref: -99 pid: 22506 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.349053:0:22506:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421ca08 lrc: 5/1,1 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc4a4 expref: -99 pid: 22506 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.349060:0:22506:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(EX) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421ca08 lrc: 5/1,1 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc4a4 expref: -99 pid: 22506 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.349068:0:22506:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421ca08 lrc: 4/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc4a4 expref: -99 pid: 22506 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764854.349078:0:22506:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8f994f2e4000 x1862022967055488/t8589934633(8589934633) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 376/896 e 0 to 0 dl 1775764870 ref 2 fl Complete:RQU/604/0 rc 0/0 job:'rwv.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764854.349098:0:22506:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421ca08 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc4a4 expref: -99 pid: 22506 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1775764854.349109:0:22506:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421ca08 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc4a4 expref: -99 pid: 22506 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.349122:0:22506:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 16 res 8589935617/3808 (0 0) 00000080:00200000:1.0:1775764854.349128:0:22506:0:(namei.c:943:ll_intent_lock()) intent lock 1024 on i1 [0x200000401:0xee0:0x0] suppgids 0 0: rc 0 00000080:00010000:1.0:1775764854.349137:0:22506:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xee0:0x0](ffff8f9952eb7448) for lock 0x36f9bdc9e421ca08 00000080:00010000:1.0:1775764854.349145:0:22506:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421ca08 from it ffffafbac1df3b00 00010000:00010000:1.0:1775764854.349147:0:22506:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421ca08 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc4a4 expref: -99 pid: 22506 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764854.349154:0:22506:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421ca08 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc4a4 expref: -99 pid: 22506 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764854.349198:0:22506:0:(file.c:2206:ll_file_io_generic()) f11a.sanity-pfl: 2 io complete with rc: 0, result: 0, restart: 1 00000080:00200000:1.0:1775764854.349202:0:22506:0:(file.c:2213:ll_file_io_generic()) f11a.sanity-pfl: restart write from ppos=2098176 bytes=1047553 retries=999 ret=0: rc = 0 00000080:00200000:1.0:1775764854.349209:0:22506:0:(file.c:1815:ll_io_set_mirror()) f11a.sanity-pfl: desiginated mirror: 0 00000020:00200000:1.0:1775764854.349217:0:22506:0:(cl_io.c:214:cl_io_rw_init()) header@000000008dd138bb[0x110, 2, [0x200000401:0xee0:0x0] hash] 00000020:00200000:1.0:1775764854.349219:0:22506:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [2098176, 3145729) 0 1 00000080:00200000:1.0:1775764854.349225:0:22506:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xee0:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00010000:00010000:1.0:1775764854.349240:0:22506:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421ca08 lrc: 2/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc4a4 expref: -99 pid: 22506 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764854.349251:0:22506:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421ca08 lrc: 2/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc4a4 expref: -99 pid: 22506 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:1.0:1775764854.349258:0:22506:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200000401:0xee0:0x0](000000002c188e9c) being reconfigured ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421ca08 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc4a4 expref: -99 pid: 22506 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764854.349340:0:22506:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421ca08 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc4a4 expref: -99 pid: 22506 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764854.349347:0:22506:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421ca08 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc4a4 expref: -99 pid: 22506 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764854.349376:0:22506:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xee0:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764854.349387:0:22506:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:1.0:1775764854.349411:0:22506:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:1.0:1775764854.349424:0:22506:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x10 00020000:00200000:1.0:1775764854.349454:0:22506:0:(lov_io.c:860:lov_io_iter_init()) component[3] flags 0x0 00020000:00200000:1.0:1775764854.349456:0:22506:0:(lov_io.c:860:lov_io_iter_init()) component[4] flags 0x0 00000008:00010000:1.0:1775764854.349472:0:22506:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### extent: 000000007bd0fffd ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000008:00010000:1.0:1775764854.349497:0:22506:0:(osc_cache.c:1167:osc_extent_make_ready()) ### extent: 000000007bd0fffd ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764854.349538:0:22506:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (2097152 2101247) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000008:00100000:1.0:1775764854.350089:0:22506:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8f994f2e7800 - object 0x280000403:1796 offset 2097152<>2098176 00000100:00100000:1.0:1775764854.350130:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1775764854.350146:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967055616:192.168.202.146@tcp:4:rwv.0 00000008:00010000:1.0:1775764854.354199:0:2379:0:(osc_cache.c:901:osc_extent_finish()) ### extent: 000000007bd0fffd ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000008:00010000:1.0:1775764854.354256:0:2379:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000007bd0fffd ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000100:00100000:1.0:1775764854.354283:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967055616:192.168.202.146@tcp:4:rwv.0 00000080:00200000:3.0:1775764854.354517:0:22506:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xee0:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1775764854.354624:0:22506:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421ca08 lrc: 2/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc4a4 expref: -99 pid: 22506 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764854.354638:0:22506:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421ca08 lrc: 2/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc4a4 expref: -99 pid: 22506 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:3.0:1775764854.354647:0:22506:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200000401:0xee0:0x0](000000002c188e9c) being reconfigured ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421ca08 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc4a4 expref: -99 pid: 22506 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764854.354730:0:22506:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xee0:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1775764854.354781:0:22506:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764854.354799:0:22506:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:3.0:1775764854.354827:0:22506:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x10 00020000:00200000:3.0:1775764854.354852:0:22506:0:(lov_io.c:860:lov_io_iter_init()) component[3] flags 0x0 00020000:00200000:3.0:1775764854.354854:0:22506:0:(lov_io.c:860:lov_io_iter_init()) component[4] flags 0x0 00000080:00200000:3.0:1775764854.354887:0:22506:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xee0:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:3.0:1775764854.354907:0:22506:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 0 blocks 1 [cma]time 0 0 0 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 1/0,0 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000008:00010000:3.0:1775764854.354929:0:22506:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 1048576 blocks 1 [cma]time: 1775764853 1775764853 0 by oinfo size 1048577 blocks 1 [cma]time 1775764854 1775764854 0 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 1/0,0 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000008:00010000:3.0:1775764854.354963:0:22506:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 2098176 blocks 1 [cma]time 1775764854 1775764854 0 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 1/0,0 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000080:00200000:3.0:1775764854.355060:0:22506:0:(file.c:6766:ll_layout_conf()) [0x200000401:0xee0:0x0]: layout version change: 4294967294 -> 8 00010000:00010000:3.0:1775764854.355071:0:22506:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421ca08 lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fc4a4 expref: -99 pid: 22506 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764854.355079:0:22506:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641a00/0x36f9bdc9e421ca08 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xee0:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fc4a4 expref: -99 pid: 22506 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764854.355098:0:22506:0:(file.c:2136:ll_file_io_generic()) Range lock [0, 18446744073709551615] 00020000:00200000:3.0:1775764854.355105:0:22506:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1775764854.355118:0:22506:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:3.0:1775764854.355126:0:22506:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x10 00020000:00200000:3.0:1775764854.355137:0:22506:0:(lov_io.c:860:lov_io_iter_init()) component[3] flags 0x10 00000080:00200000:3.0:1775764854.355151:0:22506:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [0, 1024] 00000008:00010000:3.0:1775764854.355163:0:22506:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764854.355476:0:22506:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99708df030, osc lock ffff8f9966aced10, flags 0x0 00000008:00010000:3.0:1775764854.355487:0:22506:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764854.355489:0:22506:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99708df070, osc lock ffff8f9966ace130, flags 0x0 00000008:00010000:3.0:1775764854.355492:0:22506:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764854.355494:0:22506:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99708df0b0, osc lock ffff8f9966acf0a0, flags 0x0 00000008:00010000:3.0:1775764854.355497:0:22506:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764854.355498:0:22506:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f99708df0f0, osc lock ffff8f9966acf1d0, flags 0x0 00010000:00010000:3.0:1775764854.355514:0:22506:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1775764854.355525:0:22506:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1775764854.355552:0:22506:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 6/0,2 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000008:00010000:3.0:1775764854.355561:0:22506:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 5/0,2 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1775764854.355570:0:22506:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 6/0,2 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1775764854.355576:0:22506:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1775764854.355587:0:22506:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1775764854.355595:0:22506:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (1048576 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1775764854.355603:0:22506:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 6/0,2 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000008:00010000:3.0:1775764854.355798:0:22506:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 1048577, setting rss=1048577; kms=1048577, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 5/0,2 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1775764854.355825:0:22506:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 6/0,2 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1775764854.355832:0:22506:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1775764854.355855:0:22506:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:3.0:1775764854.355863:0:22506:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (2097152 3145727) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:3.0:1775764854.355871:0:22506:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 6/0,2 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000008:00010000:3.0:1775764854.355878:0:22506:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 2098176, setting rss=2098176; kms=2098176, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 5/0,2 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:3.0:1775764854.355885:0:22506:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 6/0,2 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:3.0:1775764854.355891:0:22506:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:3.0:1775764854.355952:0:22506:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000587dd0e8/0x36f9bdc9e421ca0f lrc: 3/0,1 mode: --/PW res: [0x240000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 22506 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764854.355961:0:22506:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000587dd0e8/0x36f9bdc9e421ca0f lrc: 3/0,1 mode: --/PW res: [0x240000403:0x704:0x0].0x0 rrc: 2 type: EXT [3145728->4194303] (req 3145728->4194303) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 22506 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764854.355970:0:22506:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000587dd0e8/0x36f9bdc9e421ca0f lrc: 3/0,1 mode: --/PW res: [0x240000403:0x704:0x0].0x0 rrc: 2 type: EXT [3145728->4194303] (req 3145728->4194303) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 22506 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764854.355992:0:22506:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job rwv:3f83546c-01a9-4156-8a97-44c446245d3e:22506:1862022967055744:192.168.202.146@tcp:101:rwv.0 00000100:00100000:3.0:1775764854.356071:0:22506:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994ecbb000 going to sleep for 11 seconds 00000100:00100000:3.0:1775764854.359103:0:22506:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job rwv:3f83546c-01a9-4156-8a97-44c446245d3e:22506:1862022967055744:192.168.202.146@tcp:101:rwv.0 00010000:00010000:3.0:1775764854.359129:0:22506:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000587dd0e8/0x36f9bdc9e421ca0f lrc: 4/0,1 mode: --/PW res: [0x240000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc4ab expref: -99 pid: 22506 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764854.359147:0:22506:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000587dd0e8/0x36f9bdc9e421ca0f lrc: 4/0,1 mode: PW/PW res: [0x240000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc4ab expref: -99 pid: 22506 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764854.359159:0:22506:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000587dd0e8/0x36f9bdc9e421ca0f lrc: 4/0,1 mode: PW/PW res: [0x240000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc4ab expref: -99 pid: 22506 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764854.359183:0:22506:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000587dd0e8/0x36f9bdc9e421ca0f lrc: 5/0,2 mode: PW/PW res: [0x240000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc4ab expref: -99 pid: 22506 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:3.0:1775764854.359232:0:22506:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000587dd0e8/0x36f9bdc9e421ca0f lrc: 4/0,2 mode: PW/PW res: [0x240000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fc4ab expref: -99 pid: 22506 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764854.359244:0:22506:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000587dd0e8/0x36f9bdc9e421ca0f lrc: 5/0,2 mode: PW/PW res: [0x240000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc4ab expref: -99 pid: 22506 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764854.359251:0:22506:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000587dd0e8/0x36f9bdc9e421ca0f lrc: 4/0,1 mode: PW/PW res: [0x240000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc4ab expref: -99 pid: 22506 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:3.0:1775764854.359289:0:22506:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0xee0:0x0] updating i_size 2098176 i_blocks 4 00000080:00200000:3.0:1775764854.359294:0:22506:0:(vvp_io.c:1286:vvp_io_write_start()) f11a.sanity-pfl: write [2098176, 3145729) append 1 append lockpos 4194304 00000080:00200000:3.0:1775764854.359312:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 512 of 1024 to 3072 bytes 00000008:00010000:3.0:1775764854.359409:0:22506:0:(osc_cache.c:2998:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8f99705b9a80 00000008:00010000:3.0:1775764854.359468:0:21596:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8f99705b9a80 00010000:00010000:3.0:1775764854.359496:0:21596:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (2097152 2101247) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000008:00100000:3.0:1775764854.359532:0:21596:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8f9970eddc00 - object 0x280000403:1796 offset 2097152<>2101248 00000100:00100000:2.0:1775764854.359591:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967055872:192.168.202.146@tcp:3:rwv.0 00000008:00010000:2.0:1775764854.363570:0:2381:0:(osc_cache.c:901:osc_extent_finish()) ### no dlm lock: extent: ffff8f99705b9a80 00000020:00200000:2.0:1775764854.363586:0:2381:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1775764854.363787:0:2381:0:(osc_cache.c:496:__osc_extent_remove()) ### no dlm lock: extent: ffff8f99705b9a80 00000100:00100000:2.0:1775764854.363812:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967055872:192.168.202.146@tcp:3:rwv.0 00000080:00200000:3.0:1775764854.364010:0:22506:0:(rw26.c:858:ll_write_end()) pos 2098176, len 3072, copied 3072 00000080:00200000:3.0:1775764854.364021:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000eb96ac62[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:3.0:1775764854.364025:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000ec79eeb1 17ffffc0001009 3:0 ffff8f9970939460 512 lru 00000080:00200000:3.0:1775764854.364032:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000cefddbf4 512: 1< 1 - - > 2< 2097152 0 4096 0x0 0x40009 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:3.0:1775764854.364035:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000eb96ac62 00000080:00200000:3.0:1775764854.364036:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764854.364041:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 513 of 0 to 4096 bytes 00000080:00200000:3.0:1775764854.364084:0:22506:0:(rw26.c:858:ll_write_end()) pos 2101248, len 4096, copied 4096 00000080:00200000:3.0:1775764854.364089:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000eb88df69[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:3.0:1775764854.364091:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000d924a83e 17ffffc0001001 3:0 ffff8f99709390e0 513 lru 00000080:00200000:3.0:1775764854.364097:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000006db76206 513: 1< 0 - - > 2< 2101248 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:3.0:1775764854.364099:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000eb88df69 00000080:00200000:3.0:1775764854.364100:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 2. 00000080:00200000:3.0:1775764854.364102:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 514 of 0 to 4096 bytes 00000080:00200000:3.0:1775764854.364120:0:22506:0:(rw26.c:858:ll_write_end()) pos 2105344, len 4096, copied 4096 00000080:00200000:3.0:1775764854.364124:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000014c71c53[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:3.0:1775764854.364126:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000a413ba17 17ffffc0001001 3:0 ffff8f9970939c40 514 lru 00000080:00200000:3.0:1775764854.364130:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000005e4882de 514: 1< 0 - - > 2< 2105344 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:3.0:1775764854.364132:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000014c71c53 00000080:00200000:3.0:1775764854.364133:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 3. 00000080:00200000:3.0:1775764854.364135:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 515 of 0 to 4096 bytes 00000080:00200000:3.0:1775764854.364147:0:22506:0:(rw26.c:858:ll_write_end()) pos 2109440, len 4096, copied 4096 00000080:00200000:3.0:1775764854.364151:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000155d6953[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:3.0:1775764854.364153:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000029627804 17ffffc0001001 3:0 ffff8f9970939a80 515 lru 00000080:00200000:3.0:1775764854.364158:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000090c25fa2 515: 1< 0 - - > 2< 2109440 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:3.0:1775764854.364160:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000155d6953 00000080:00200000:3.0:1775764854.364161:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 4. 00000080:00200000:3.0:1775764854.364163:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 516 of 0 to 4096 bytes 00000080:00200000:3.0:1775764854.364254:0:22506:0:(rw26.c:858:ll_write_end()) pos 2113536, len 4096, copied 4096 00000080:00200000:3.0:1775764854.364258:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000adcf1b78[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:3.0:1775764854.364261:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000009ec5a020 17ffffc0001001 3:0 ffff8f99709398c0 516 lru 00000080:00200000:3.0:1775764854.364266:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000002147d43b 516: 1< 0 - - > 2< 2113536 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:3.0:1775764854.364268:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000adcf1b78 00000080:00200000:3.0:1775764854.364269:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 5. 00000080:00200000:3.0:1775764854.364270:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 517 of 0 to 4096 bytes 00000080:00200000:3.0:1775764854.364284:0:22506:0:(rw26.c:858:ll_write_end()) pos 2117632, len 4096, copied 4096 00000080:00200000:3.0:1775764854.364287:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000007baec74[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:3.0:1775764854.364290:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000a3e96d2c 17ffffc0001001 3:0 ffff8f9970939d20 517 lru 00000080:00200000:3.0:1775764854.364295:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000003a8802cb 517: 1< 0 - - > 2< 2117632 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:3.0:1775764854.364297:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000007baec74 00000080:00200000:3.0:1775764854.364298:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 6. 00000080:00200000:3.0:1775764854.364300:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 518 of 0 to 4096 bytes 00000080:00200000:3.0:1775764854.364313:0:22506:0:(rw26.c:858:ll_write_end()) pos 2121728, len 4096, copied 4096 00000080:00200000:3.0:1775764854.364317:0:22506:0:(rw26.c:916:ll_write_end()) page@000000006b375b39[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:3.0:1775764854.364319:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000009d9b73f2 17ffffc0001001 3:0 ffff8f99709392a0 518 lru 00000080:00200000:3.0:1775764854.364324:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000006bc892d7 518: 1< 0 - - > 2< 2121728 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:3.0:1775764854.364326:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000006b375b39 00000080:00200000:3.0:1775764854.364327:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 7. 00000080:00200000:3.0:1775764854.364329:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 519 of 0 to 4096 bytes 00000080:00200000:3.0:1775764854.364339:0:22506:0:(rw26.c:858:ll_write_end()) pos 2125824, len 4096, copied 4096 00000080:00200000:3.0:1775764854.364343:0:22506:0:(rw26.c:916:ll_write_end()) page@000000009875875d[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:3.0:1775764854.364345:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000d4e67c74 17ffffc0001001 3:0 ffff8f9970939620 519 lru 00000080:00200000:3.0:1775764854.364350:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000088f8aae4 519: 1< 0 - - > 2< 2125824 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:3.0:1775764854.364352:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000009875875d 00000080:00200000:3.0:1775764854.364354:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 8. 00000080:00200000:3.0:1775764854.364356:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 520 of 0 to 4096 bytes 00000080:00200000:3.0:1775764854.364368:0:22506:0:(rw26.c:858:ll_write_end()) pos 2129920, len 4096, copied 4096 00000080:00200000:3.0:1775764854.364371:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000054664cc4[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:3.0:1775764854.364373:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000e032eaac 17ffffc0001001 3:0 ffff8f9970939700 520 lru 00000080:00200000:3.0:1775764854.364377:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000008496b1c2 520: 1< 0 - - > 2< 2129920 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:3.0:1775764854.364379:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000054664cc4 00000080:00200000:3.0:1775764854.364380:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 9. 00000080:00200000:3.0:1775764854.364383:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 521 of 0 to 4096 bytes 00000080:00200000:3.0:1775764854.364396:0:22506:0:(rw26.c:858:ll_write_end()) pos 2134016, len 4096, copied 4096 00000080:00200000:3.0:1775764854.364400:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000fd709e34[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:3.0:1775764854.364402:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000024fdaedf 17ffffc0001001 3:0 ffff8f9970939e00 521 lru 00000080:00200000:3.0:1775764854.364406:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000c14de22d 521: 1< 0 - - > 2< 2134016 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:3.0:1775764854.364408:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000fd709e34 00000080:00200000:3.0:1775764854.364409:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 10. 00000080:00200000:3.0:1775764854.364411:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 522 of 0 to 4096 bytes 00000080:00200000:3.0:1775764854.364421:0:22506:0:(rw26.c:858:ll_write_end()) pos 2138112, len 4096, copied 4096 00000080:00200000:3.0:1775764854.364425:0:22506:0:(rw26.c:916:ll_write_end()) page@000000005d6823fc[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:3.0:1775764854.364427:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000041a3c9d9 17ffffc0001001 3:0 ffff8f99709397e0 522 lru 00000080:00200000:3.0:1775764854.364432:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000f91a776d 522: 1< 0 - - > 2< 2138112 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:3.0:1775764854.364435:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000005d6823fc 00000080:00200000:3.0:1775764854.364436:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 11. 00000080:00200000:3.0:1775764854.364438:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 523 of 0 to 4096 bytes 00000080:00200000:3.0:1775764854.364452:0:22506:0:(rw26.c:858:ll_write_end()) pos 2142208, len 4096, copied 4096 00000080:00200000:3.0:1775764854.364456:0:22506:0:(rw26.c:916:ll_write_end()) page@000000006521a2cb[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:3.0:1775764854.364458:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000b0292bc1 17ffffc0001001 3:0 ffff8f99709391c0 523 lru 00000080:00200000:3.0:1775764854.364462:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000002ac52950 523: 1< 0 - - > 2< 2142208 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:3.0:1775764854.364464:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000006521a2cb 00000080:00200000:3.0:1775764854.364465:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 12. 00000080:00200000:3.0:1775764854.364467:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 524 of 0 to 4096 bytes 00000080:00200000:3.0:1775764854.364481:0:22506:0:(rw26.c:858:ll_write_end()) pos 2146304, len 4096, copied 4096 00000080:00200000:3.0:1775764854.364485:0:22506:0:(rw26.c:916:ll_write_end()) page@000000004480c78c[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:3.0:1775764854.364487:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000a6e7a35e 17ffffc0001001 3:0 ffff8f99709399a0 524 lru 00000080:00200000:3.0:1775764854.364492:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000e00d77ba 524: 1< 0 - - > 2< 2146304 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:3.0:1775764854.364494:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000004480c78c 00000080:00200000:3.0:1775764854.364494:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 13. 00000080:00200000:3.0:1775764854.364497:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 525 of 0 to 4096 bytes 00000080:00200000:3.0:1775764854.364510:0:22506:0:(rw26.c:858:ll_write_end()) pos 2150400, len 4096, copied 4096 00000080:00200000:3.0:1775764854.364514:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000430bd191[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:3.0:1775764854.364516:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000008b5dfc8 17ffffc0001001 3:0 ffff8f9970939ee0 525 lru 00000080:00200000:3.0:1775764854.364521:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000c7917c13 525: 1< 0 - - > 2< 2150400 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:3.0:1775764854.364523:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000430bd191 00000080:00200000:3.0:1775764854.364524:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 14. 00000080:00200000:3.0:1775764854.364526:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 526 of 0 to 4096 bytes 00000080:00200000:3.0:1775764854.364547:0:22506:0:(rw26.c:858:ll_write_end()) pos 2154496, len 4096, copied 4096 00000080:00200000:3.0:1775764854.364551:0:22506:0:(rw26.c:916:ll_write_end()) page@000000001f63de42[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:3.0:1775764854.364554:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000000cca0423 17ffffc0001021 2:0 ffff8f9970939b60 526 lru 00000080:00200000:3.0:1775764854.364559:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000ca4ab9fa 526: 1< 0 - - > 2< 2154496 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:3.0:1775764854.364561:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000001f63de42 00000080:00200000:3.0:1775764854.364562:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 15. 00000080:00200000:3.0:1775764854.364564:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 527 of 0 to 4096 bytes 00000080:00200000:3.0:1775764854.364578:0:22506:0:(rw26.c:858:ll_write_end()) pos 2158592, len 4096, copied 4096 00000080:00200000:3.0:1775764854.364582:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000099cd17b2[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:3.0:1775764854.364585:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000e4b683ec 17ffffc0001001 3:0 ffff8f9970939540 527 lru 00000080:00200000:3.0:1775764854.364589:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000007b47aaf5 527: 1< 0 - - > 2< 2158592 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:3.0:1775764854.364591:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000099cd17b2 00000080:00200000:3.0:1775764854.364592:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 16. 00000080:00200000:3.0:1775764854.364594:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 528 of 0 to 4096 bytes 00000080:00200000:3.0:1775764854.364700:0:22506:0:(rw26.c:858:ll_write_end()) pos 2162688, len 4096, copied 4096 00000080:00200000:3.0:1775764854.364706:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000bec1f162[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:3.0:1775764854.364709:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000007c6d7b0c 17ffffc0001001 3:0 ffff8f9970939000 528 lru 00000080:00200000:3.0:1775764854.364714:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000006b912618 528: 1< 0 - - > 2< 2162688 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:3.0:1775764854.364716:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000bec1f162 00000080:00200000:3.0:1775764854.364717:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 17. 00000080:00200000:3.0:1775764854.364720:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 529 of 0 to 4096 bytes 00000080:00200000:3.0:1775764854.364735:0:22506:0:(rw26.c:858:ll_write_end()) pos 2166784, len 4096, copied 4096 00000080:00200000:3.0:1775764854.364815:0:22506:0:(rw26.c:916:ll_write_end()) page@000000002fcc9a7a[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:3.0:1775764854.364817:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000048f7db32 17ffffc0001001 3:0 ffff8f9970939380 529 lru 00000080:00200000:3.0:1775764854.364822:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000027168db7 529: 1< 0 - - > 2< 2166784 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:3.0:1775764854.364823:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000002fcc9a7a 00000080:00200000:3.0:1775764854.364824:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 18. 00000080:00200000:3.0:1775764854.364826:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 530 of 0 to 4096 bytes 00000080:00200000:3.0:1775764854.364880:0:22506:0:(rw26.c:858:ll_write_end()) pos 2170880, len 4096, copied 4096 00000080:00200000:3.0:1775764854.364885:0:22506:0:(rw26.c:916:ll_write_end()) page@000000005272361a[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:3.0:1775764854.364887:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000005ba8fcfa 17ffffc0001001 3:0 ffff8f9970f37540 530 lru 00000080:00200000:3.0:1775764854.364892:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000019cb8a3b 530: 1< 0 - - > 2< 2170880 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:3.0:1775764854.364894:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000005272361a 00000080:00200000:3.0:1775764854.364895:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 19. 00000080:00200000:3.0:1775764854.364897:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 531 of 0 to 4096 bytes 00000080:00200000:3.0:1775764854.364929:0:22506:0:(rw26.c:858:ll_write_end()) pos 2174976, len 4096, copied 4096 00000080:00200000:3.0:1775764854.364933:0:22506:0:(rw26.c:916:ll_write_end()) page@000000003524e7c7[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:3.0:1775764854.364936:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000008377a443 17ffffc0001001 3:0 ffff8f9970f37460 531 lru 00000080:00200000:3.0:1775764854.364941:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000637675eb 531: 1< 0 - - > 2< 2174976 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:3.0:1775764854.364944:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000003524e7c7 00000080:00200000:3.0:1775764854.364944:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 20. 00000080:00200000:3.0:1775764854.364946:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 532 of 0 to 4096 bytes 00000080:00200000:3.0:1775764854.364960:0:22506:0:(rw26.c:858:ll_write_end()) pos 2179072, len 4096, copied 4096 00000080:00200000:3.0:1775764854.364964:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000e201c476[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:3.0:1775764854.364966:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000e0cc7e74 17ffffc0001001 3:0 ffff8f9970f37000 532 lru 00000080:00200000:3.0:1775764854.364972:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000de251730 532: 1< 0 - - > 2< 2179072 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:3.0:1775764854.364974:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000e201c476 00000080:00200000:3.0:1775764854.364974:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 21. 00000080:00200000:3.0:1775764854.364977:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 533 of 0 to 4096 bytes 00000080:00200000:3.0:1775764854.364992:0:22506:0:(rw26.c:858:ll_write_end()) pos 2183168, len 4096, copied 4096 00000080:00200000:3.0:1775764854.364995:0:22506:0:(rw26.c:916:ll_write_end()) page@000000006f3b2868[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:3.0:1775764854.364997:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000038febd70 17ffffc0001001 3:0 ffff8f9970f371c0 533 lru 00000080:00200000:3.0:1775764854.365002:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000009c9dffe2 533: 1< 0 - - > 2< 2183168 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:3.0:1775764854.365004:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000006f3b2868 00000080:00200000:3.0:1775764854.365005:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 22. 00000080:00200000:3.0:1775764854.365007:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 534 of 0 to 4096 bytes 00000080:00200000:3.0:1775764854.365022:0:22506:0:(rw26.c:858:ll_write_end()) pos 2187264, len 4096, copied 4096 00000080:00200000:3.0:1775764854.365025:0:22506:0:(rw26.c:916:ll_write_end()) page@000000000111a83e[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:3.0:1775764854.365028:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000007db64922 17ffffc0001001 3:0 ffff8f9970f37620 534 lru 00000080:00200000:3.0:1775764854.365033:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000049109627 534: 1< 0 - - > 2< 2187264 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:3.0:1775764854.365035:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000000111a83e 00000080:00200000:3.0:1775764854.365036:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 23. 00000080:00200000:3.0:1775764854.365038:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 535 of 0 to 4096 bytes 00000080:00200000:3.0:1775764854.365054:0:22506:0:(rw26.c:858:ll_write_end()) pos 2191360, len 4096, copied 4096 00000080:00200000:3.0:1775764854.365058:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000a5566936[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:3.0:1775764854.365061:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000008ebe6269 17ffffc0001001 3:0 ffff8f9970f37e00 535 lru 00000080:00200000:3.0:1775764854.365066:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000014edc538 535: 1< 0 - - > 2< 2191360 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:3.0:1775764854.365081:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000a5566936 00000080:00200000:3.0:1775764854.365082:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 24. 00000080:00200000:3.0:1775764854.365084:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 536 of 0 to 4096 bytes 00000080:00200000:3.0:1775764854.365099:0:22506:0:(rw26.c:858:ll_write_end()) pos 2195456, len 4096, copied 4096 00000080:00200000:3.0:1775764854.365102:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000dce1ee5b[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:3.0:1775764854.365105:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000009e6b64f7 17ffffc0001001 3:0 ffff8f9970f37c40 536 lru 00000080:00200000:3.0:1775764854.365111:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000bcbd4c7e 536: 1< 0 - - > 2< 2195456 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:3.0:1775764854.365113:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000dce1ee5b 00000080:00200000:3.0:1775764854.365114:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 25. 00000080:00200000:3.0:1775764854.365116:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 537 of 0 to 4096 bytes 00000080:00200000:3.0:1775764854.365131:0:22506:0:(rw26.c:858:ll_write_end()) pos 2199552, len 4096, copied 4096 00000080:00200000:3.0:1775764854.365134:0:22506:0:(rw26.c:916:ll_write_end()) page@000000005d130819[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:3.0:1775764854.365137:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000006dd7d7c0 17ffffc0001001 3:0 ffff8f9970f378c0 537 lru 00000080:00200000:3.0:1775764854.365142:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000079d5c3e0 537: 1< 0 - - > 2< 2199552 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:3.0:1775764854.365144:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000005d130819 00000080:00200000:3.0:1775764854.365146:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 26. 00000080:00200000:3.0:1775764854.365147:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 538 of 0 to 4096 bytes 00000080:00200000:3.0:1775764854.365396:0:22506:0:(rw26.c:858:ll_write_end()) pos 2203648, len 4096, copied 4096 00000080:00200000:3.0:1775764854.365401:0:22506:0:(rw26.c:916:ll_write_end()) page@000000009f778e83[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:3.0:1775764854.365404:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000af178a26 17ffffc0001001 3:0 ffff8f9970f37d20 538 lru 00000080:00200000:3.0:1775764854.365410:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000011e257fa 538: 1< 0 - - > 2< 2203648 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:3.0:1775764854.365411:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000009f778e83 00000080:00200000:3.0:1775764854.365413:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 27. 00000080:00200000:3.0:1775764854.365415:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 539 of 0 to 4096 bytes 00000080:00200000:3.0:1775764854.365432:0:22506:0:(rw26.c:858:ll_write_end()) pos 2207744, len 4096, copied 4096 00000080:00200000:3.0:1775764854.365436:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000b9bb2217[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:3.0:1775764854.365439:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000e0c9adf0 17ffffc0001001 3:0 ffff8f9970f370e0 539 lru 00000080:00200000:3.0:1775764854.365444:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000b5781026 539: 1< 0 - - > 2< 2207744 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:3.0:1775764854.365446:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000b9bb2217 00000080:00200000:3.0:1775764854.365447:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 28. 00000080:00200000:3.0:1775764854.365449:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 540 of 0 to 4096 bytes 00000080:00200000:3.0:1775764854.365478:0:22506:0:(rw26.c:858:ll_write_end()) pos 2211840, len 4096, copied 4096 00000080:00200000:3.0:1775764854.365482:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000d411d729[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:3.0:1775764854.365485:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000006c56fbb5 17ffffc0001001 3:0 ffff8f9970f372a0 540 lru 00000080:00200000:3.0:1775764854.365490:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000004b5400e0 540: 1< 0 - - > 2< 2211840 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:3.0:1775764854.365492:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000d411d729 00000080:00200000:3.0:1775764854.365493:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 29. 00000080:00200000:3.0:1775764854.365495:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 541 of 0 to 4096 bytes 00000080:00200000:3.0:1775764854.365514:0:22506:0:(rw26.c:858:ll_write_end()) pos 2215936, len 4096, copied 4096 00000080:00200000:3.0:1775764854.365518:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000ee67159e[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:3.0:1775764854.365521:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000ee435b7e 17ffffc0001021 2:0 ffff8f9970f37380 541 lru 00000080:00200000:3.0:1775764854.365526:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000e67278e7 541: 1< 0 - - > 2< 2215936 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:3.0:1775764854.365528:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000ee67159e 00000080:00200000:3.0:1775764854.365529:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 30. 00000080:00200000:3.0:1775764854.365531:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 542 of 0 to 4096 bytes 00000080:00200000:3.0:1775764854.365547:0:22506:0:(rw26.c:858:ll_write_end()) pos 2220032, len 4096, copied 4096 00000080:00200000:3.0:1775764854.365551:0:22506:0:(rw26.c:916:ll_write_end()) page@000000009764c98e[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:3.0:1775764854.365553:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000df0621de 17ffffc0001001 3:0 ffff8f9970f37b60 542 lru 00000080:00200000:3.0:1775764854.365559:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000078677334 542: 1< 0 - - > 2< 2220032 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:3.0:1775764854.365560:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000009764c98e 00000080:00200000:3.0:1775764854.365561:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 31. 00000080:00200000:3.0:1775764854.365563:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 543 of 0 to 4096 bytes 00000080:00200000:3.0:1775764854.365578:0:22506:0:(rw26.c:858:ll_write_end()) pos 2224128, len 4096, copied 4096 00000080:00200000:3.0:1775764854.365581:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000045327907[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:3.0:1775764854.365584:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000077ee07b2 17ffffc0001001 3:0 ffff8f9970f37700 543 lru 00000080:00200000:3.0:1775764854.365588:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000001932d778 543: 1< 0 - - > 2< 2224128 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:3.0:1775764854.365590:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000045327907 00000080:00200000:3.0:1775764854.365591:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 32. 00000080:00200000:3.0:1775764854.365592:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 544 of 0 to 4096 bytes 00000080:00200000:3.0:1775764854.365607:0:22506:0:(rw26.c:858:ll_write_end()) pos 2228224, len 4096, copied 4096 00000080:00200000:3.0:1775764854.365709:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000c1b3c846[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:3.0:1775764854.365713:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000002553a107 17ffffc0001001 3:0 ffff8f9970f377e0 544 lru 00000080:00200000:3.0:1775764854.365719:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000bb3e2dfc 544: 1< 0 - - > 2< 2228224 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:3.0:1775764854.365735:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000c1b3c846 00000080:00200000:3.0:1775764854.365736:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 33. 00000080:00200000:2.0:1775764854.365918:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 545 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.365937:0:22506:0:(rw26.c:858:ll_write_end()) pos 2232320, len 4096, copied 4096 00000080:00200000:2.0:1775764854.365940:0:22506:0:(rw26.c:916:ll_write_end()) page@000000008d7d0e42[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.365943:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000029f67c1 17ffffc0001001 3:0 ffff8f99706752a0 545 lru 00000080:00200000:2.0:1775764854.365948:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000a6685386 545: 1< 0 - - > 2< 2232320 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.365950:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000008d7d0e42 00000080:00200000:2.0:1775764854.365951:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 34. 00000080:00200000:2.0:1775764854.365953:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 546 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.365964:0:22506:0:(rw26.c:858:ll_write_end()) pos 2236416, len 4096, copied 4096 00000080:00200000:2.0:1775764854.365967:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000276e446b[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.365970:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000f644c475 17ffffc0001001 3:0 ffff8f9970675d20 546 lru 00000080:00200000:2.0:1775764854.365974:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000f9353da8 546: 1< 0 - - > 2< 2236416 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.365976:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000276e446b 00000080:00200000:2.0:1775764854.365977:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 35. 00000080:00200000:2.0:1775764854.365978:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 547 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.365989:0:22506:0:(rw26.c:858:ll_write_end()) pos 2240512, len 4096, copied 4096 00000080:00200000:2.0:1775764854.365992:0:22506:0:(rw26.c:916:ll_write_end()) page@000000005a85d445[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.365994:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000e0c9ebbb 17ffffc0001001 3:0 ffff8f9970675460 547 lru 00000080:00200000:2.0:1775764854.365999:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000099d44655 547: 1< 0 - - > 2< 2240512 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.366001:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000005a85d445 00000080:00200000:2.0:1775764854.366002:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 36. 00000080:00200000:2.0:1775764854.366003:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 548 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.366015:0:22506:0:(rw26.c:858:ll_write_end()) pos 2244608, len 4096, copied 4096 00000080:00200000:2.0:1775764854.366019:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000ad541fcd[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.366021:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000006271eda7 17ffffc0001001 3:0 ffff8f9970675000 548 lru 00000080:00200000:2.0:1775764854.366026:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000b076f704 548: 1< 0 - - > 2< 2244608 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.366028:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000ad541fcd 00000080:00200000:2.0:1775764854.366029:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 37. 00000080:00200000:2.0:1775764854.366031:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 549 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.366042:0:22506:0:(rw26.c:858:ll_write_end()) pos 2248704, len 4096, copied 4096 00000080:00200000:2.0:1775764854.366045:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000040892221[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.366047:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000000498bef0 17ffffc0001001 3:0 ffff8f9970675620 549 lru 00000080:00200000:2.0:1775764854.366052:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000cb05e1d3 549: 1< 0 - - > 2< 2248704 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.366054:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000040892221 00000080:00200000:2.0:1775764854.366055:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 38. 00000080:00200000:2.0:1775764854.366056:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 550 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.366069:0:22506:0:(rw26.c:858:ll_write_end()) pos 2252800, len 4096, copied 4096 00000080:00200000:2.0:1775764854.366072:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000043aafd77[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.366075:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000009366fc41 17ffffc0001001 3:0 ffff8f9970675c40 550 lru 00000080:00200000:2.0:1775764854.366080:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000db7017c0 550: 1< 0 - - > 2< 2252800 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.366082:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000043aafd77 00000080:00200000:2.0:1775764854.366083:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 39. 00000080:00200000:2.0:1775764854.366084:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 551 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.366096:0:22506:0:(rw26.c:858:ll_write_end()) pos 2256896, len 4096, copied 4096 00000080:00200000:2.0:1775764854.366099:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000099759ce7[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.366101:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000bf222135 17ffffc0001001 3:0 ffff8f99706758c0 551 lru 00000080:00200000:2.0:1775764854.366105:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000003e0c5913 551: 1< 0 - - > 2< 2256896 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.366106:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000099759ce7 00000080:00200000:2.0:1775764854.366107:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 40. 00000080:00200000:2.0:1775764854.366109:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 552 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.366145:0:22506:0:(rw26.c:858:ll_write_end()) pos 2260992, len 4096, copied 4096 00000080:00200000:2.0:1775764854.366149:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000cb0604ea[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.366151:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000048f82cdc 17ffffc0001021 2:0 ffff8f99706751c0 552 lru 00000080:00200000:2.0:1775764854.366156:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000000d0e6551 552: 1< 0 - - > 2< 2260992 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.366158:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000cb0604ea 00000080:00200000:2.0:1775764854.366159:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 41. 00000080:00200000:2.0:1775764854.366160:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 553 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.366256:0:22506:0:(rw26.c:858:ll_write_end()) pos 2265088, len 4096, copied 4096 00000080:00200000:2.0:1775764854.366262:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000dd9fff64[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.366264:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000002967cec1 17ffffc0001001 3:0 ffff8f9970675a80 553 lru 00000080:00200000:2.0:1775764854.366269:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000051f7c508 553: 1< 0 - - > 2< 2265088 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.366271:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000dd9fff64 00000080:00200000:2.0:1775764854.366272:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 42. 00000080:00200000:2.0:1775764854.366274:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 554 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.366290:0:22506:0:(rw26.c:858:ll_write_end()) pos 2269184, len 4096, copied 4096 00000080:00200000:2.0:1775764854.366294:0:22506:0:(rw26.c:916:ll_write_end()) page@000000008ab547e1[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.366296:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000003aac165c 17ffffc0001001 3:0 ffff8f9970675ee0 554 lru 00000080:00200000:2.0:1775764854.366301:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000f47963ef 554: 1< 0 - - > 2< 2269184 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.366303:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000008ab547e1 00000080:00200000:2.0:1775764854.366304:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 43. 00000080:00200000:2.0:1775764854.366305:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 555 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.366320:0:22506:0:(rw26.c:858:ll_write_end()) pos 2273280, len 4096, copied 4096 00000080:00200000:2.0:1775764854.366323:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000045b19229[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.366326:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000039814d20 17ffffc0001001 3:0 ffff8f99706759a0 555 lru 00000080:00200000:2.0:1775764854.366331:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000981e4008 555: 1< 0 - - > 2< 2273280 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.366333:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000045b19229 00000080:00200000:2.0:1775764854.366334:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 44. 00000080:00200000:2.0:1775764854.366336:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 556 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.366350:0:22506:0:(rw26.c:858:ll_write_end()) pos 2277376, len 4096, copied 4096 00000080:00200000:2.0:1775764854.366354:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000028537273[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.366356:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000000ae0f6c 17ffffc0001001 3:0 ffff8f99706757e0 556 lru 00000080:00200000:2.0:1775764854.366362:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000ac68e01d 556: 1< 0 - - > 2< 2277376 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.366364:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000028537273 00000080:00200000:2.0:1775764854.366365:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 45. 00000080:00200000:2.0:1775764854.366367:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 557 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.366379:0:22506:0:(rw26.c:858:ll_write_end()) pos 2281472, len 4096, copied 4096 00000080:00200000:2.0:1775764854.366382:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000ae1b8afe[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.366385:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000000a5f3254 17ffffc0001001 3:0 ffff8f9970675700 557 lru 00000080:00200000:2.0:1775764854.366390:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000b905482b 557: 1< 0 - - > 2< 2281472 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.366392:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000ae1b8afe 00000080:00200000:2.0:1775764854.366393:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 46. 00000080:00200000:2.0:1775764854.366395:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 558 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.366409:0:22506:0:(rw26.c:858:ll_write_end()) pos 2285568, len 4096, copied 4096 00000080:00200000:2.0:1775764854.366412:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000061e9fb1d[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.366414:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000bb75850c 17ffffc0001001 3:0 ffff8f99706750e0 558 lru 00000080:00200000:2.0:1775764854.366419:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000ecdc64ac 558: 1< 0 - - > 2< 2285568 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.366421:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000061e9fb1d 00000080:00200000:2.0:1775764854.366422:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 47. 00000080:00200000:2.0:1775764854.366424:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 559 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.366437:0:22506:0:(rw26.c:858:ll_write_end()) pos 2289664, len 4096, copied 4096 00000080:00200000:2.0:1775764854.366441:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000009d6f666[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.366444:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000cf26c09f 17ffffc0001001 3:0 ffff8f9970675b60 559 lru 00000080:00200000:2.0:1775764854.366449:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000500b3338 559: 1< 0 - - > 2< 2289664 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.366451:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000009d6f666 00000080:00200000:2.0:1775764854.366452:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 48. 00000080:00200000:2.0:1775764854.366454:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 560 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.366468:0:22506:0:(rw26.c:858:ll_write_end()) pos 2293760, len 4096, copied 4096 00000080:00200000:2.0:1775764854.366471:0:22506:0:(rw26.c:916:ll_write_end()) page@000000003451d501[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.366474:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000002e4f4a06 17ffffc0001001 3:0 ffff8f9970675380 560 lru 00000080:00200000:2.0:1775764854.366479:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000035331de4 560: 1< 0 - - > 2< 2293760 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.366481:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000003451d501 00000080:00200000:2.0:1775764854.366482:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 49. 00000080:00200000:2.0:1775764854.366484:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 561 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.366499:0:22506:0:(rw26.c:858:ll_write_end()) pos 2297856, len 4096, copied 4096 00000080:00200000:2.0:1775764854.366503:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000fc89cfa4[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.366506:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000e0773b4f 17ffffc0001001 3:0 ffff8f9970675e00 561 lru 00000080:00200000:2.0:1775764854.366510:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000009d7d1f40 561: 1< 0 - - > 2< 2297856 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.366512:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000fc89cfa4 00000080:00200000:2.0:1775764854.366513:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 50. 00000080:00200000:2.0:1775764854.366516:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 562 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.366530:0:22506:0:(rw26.c:858:ll_write_end()) pos 2301952, len 4096, copied 4096 00000080:00200000:2.0:1775764854.366534:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000022ce2836[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.366537:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000949dc0f9 17ffffc0001001 3:0 ffff8f9970675540 562 lru 00000080:00200000:2.0:1775764854.366542:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000015fc40bb 562: 1< 0 - - > 2< 2301952 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.366545:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000022ce2836 00000080:00200000:2.0:1775764854.366546:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 51. 00000080:00200000:2.0:1775764854.366548:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 563 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.366582:0:22506:0:(rw26.c:858:ll_write_end()) pos 2306048, len 4096, copied 4096 00000080:00200000:2.0:1775764854.366587:0:22506:0:(rw26.c:916:ll_write_end()) page@000000006735b66d[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.366589:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000007a447818 17ffffc0001001 3:0 ffff8f994f1c2380 563 lru 00000080:00200000:2.0:1775764854.366594:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000009fa706a4 563: 1< 0 - - > 2< 2306048 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.366596:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000006735b66d 00000080:00200000:2.0:1775764854.366597:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 52. 00000080:00200000:2.0:1775764854.366599:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 564 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.366724:0:22506:0:(rw26.c:858:ll_write_end()) pos 2310144, len 4096, copied 4096 00000080:00200000:2.0:1775764854.366729:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000bb7bc5bb[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.366732:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000037bbd8ec 17ffffc0001001 3:0 ffff8f994f1c2b60 564 lru 00000080:00200000:2.0:1775764854.366826:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000066258e1e 564: 1< 0 - - > 2< 2310144 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.366829:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000bb7bc5bb 00000080:00200000:2.0:1775764854.366830:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 53. 00000080:00200000:2.0:1775764854.366833:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 565 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.366860:0:22506:0:(rw26.c:858:ll_write_end()) pos 2314240, len 4096, copied 4096 00000080:00200000:2.0:1775764854.366864:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000097f5c658[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.366867:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000001faf64c5 17ffffc0001001 3:0 ffff8f994f1c2700 565 lru 00000080:00200000:2.0:1775764854.366872:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000002dbada33 565: 1< 0 - - > 2< 2314240 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.366873:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000097f5c658 00000080:00200000:2.0:1775764854.366874:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 54. 00000080:00200000:2.0:1775764854.366876:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 566 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.366889:0:22506:0:(rw26.c:858:ll_write_end()) pos 2318336, len 4096, copied 4096 00000080:00200000:2.0:1775764854.366892:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000696b5baa[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.366894:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000bb047472 17ffffc0001001 3:0 ffff8f994f1c27e0 566 lru 00000080:00200000:2.0:1775764854.366899:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000793addd9 566: 1< 0 - - > 2< 2318336 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.366901:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000696b5baa 00000080:00200000:2.0:1775764854.366902:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 55. 00000080:00200000:2.0:1775764854.366903:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 567 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.366920:0:22506:0:(rw26.c:858:ll_write_end()) pos 2322432, len 4096, copied 4096 00000080:00200000:2.0:1775764854.366924:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000cad13069[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.366926:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000231c6178 17ffffc0001021 2:0 ffff8f994f1c22a0 567 lru 00000080:00200000:2.0:1775764854.366931:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000069285f29 567: 1< 0 - - > 2< 2322432 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.366933:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000cad13069 00000080:00200000:2.0:1775764854.366934:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 56. 00000080:00200000:2.0:1775764854.366936:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 568 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.366947:0:22506:0:(rw26.c:858:ll_write_end()) pos 2326528, len 4096, copied 4096 00000080:00200000:2.0:1775764854.366950:0:22506:0:(rw26.c:916:ll_write_end()) page@000000004fd3e447[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.366953:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000c573ba7f 17ffffc0001001 3:0 ffff8f994f1c20e0 568 lru 00000080:00200000:2.0:1775764854.366958:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000006a323b8 568: 1< 0 - - > 2< 2326528 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.366960:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000004fd3e447 00000080:00200000:2.0:1775764854.366961:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 57. 00000080:00200000:2.0:1775764854.366963:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 569 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.366973:0:22506:0:(rw26.c:858:ll_write_end()) pos 2330624, len 4096, copied 4096 00000080:00200000:2.0:1775764854.366977:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000b8a66eb1[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.366979:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000287506ac 17ffffc0001001 3:0 ffff8f994f1c2d20 569 lru 00000080:00200000:2.0:1775764854.366984:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000d98d2ac9 569: 1< 0 - - > 2< 2330624 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.366986:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000b8a66eb1 00000080:00200000:2.0:1775764854.366988:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 58. 00000080:00200000:2.0:1775764854.366989:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 570 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.367002:0:22506:0:(rw26.c:858:ll_write_end()) pos 2334720, len 4096, copied 4096 00000080:00200000:2.0:1775764854.367006:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000665f49b6[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.367008:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000004d60869a 17ffffc0001001 3:0 ffff8f994f1c28c0 570 lru 00000080:00200000:2.0:1775764854.367012:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000003c413692 570: 1< 0 - - > 2< 2334720 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.367014:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000665f49b6 00000080:00200000:2.0:1775764854.367015:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 59. 00000080:00200000:2.0:1775764854.367017:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 571 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.367027:0:22506:0:(rw26.c:858:ll_write_end()) pos 2338816, len 4096, copied 4096 00000080:00200000:2.0:1775764854.367030:0:22506:0:(rw26.c:916:ll_write_end()) page@000000000b1652e3[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.367032:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000c41980d9 17ffffc0001001 3:0 ffff8f994f1c2c40 571 lru 00000080:00200000:2.0:1775764854.367037:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000b7f6deec 571: 1< 0 - - > 2< 2338816 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.367039:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000000b1652e3 00000080:00200000:2.0:1775764854.367040:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 60. 00000080:00200000:2.0:1775764854.367042:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 572 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.367059:0:22506:0:(rw26.c:858:ll_write_end()) pos 2342912, len 4096, copied 4096 00000080:00200000:2.0:1775764854.367062:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000a84847aa[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.367065:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000006ca0c0bb 17ffffc0001001 3:0 ffff8f994f1c2e00 572 lru 00000080:00200000:2.0:1775764854.367070:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000055c8ae4c 572: 1< 0 - - > 2< 2342912 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.367072:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000a84847aa 00000080:00200000:2.0:1775764854.367073:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 61. 00000080:00200000:2.0:1775764854.367075:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 573 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.367089:0:22506:0:(rw26.c:858:ll_write_end()) pos 2347008, len 4096, copied 4096 00000080:00200000:2.0:1775764854.367093:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000efd93515[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.367095:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000099356905 17ffffc0001001 3:0 ffff8f994f1c2620 573 lru 00000080:00200000:2.0:1775764854.367101:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000042a51a26 573: 1< 0 - - > 2< 2347008 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.367103:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000efd93515 00000080:00200000:2.0:1775764854.367104:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 62. 00000080:00200000:2.0:1775764854.367106:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 574 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.367121:0:22506:0:(rw26.c:858:ll_write_end()) pos 2351104, len 4096, copied 4096 00000080:00200000:2.0:1775764854.367125:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000bcef50cf[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.367128:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000f41ee3c4 17ffffc0001001 3:0 ffff8f994f1c2a80 574 lru 00000080:00200000:2.0:1775764854.367133:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000006f47ee80 574: 1< 0 - - > 2< 2351104 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.367135:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000bcef50cf 00000080:00200000:2.0:1775764854.367137:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 63. 00000080:00200000:2.0:1775764854.367139:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 575 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.367152:0:22506:0:(rw26.c:858:ll_write_end()) pos 2355200, len 4096, copied 4096 00000080:00200000:2.0:1775764854.367156:0:22506:0:(rw26.c:916:ll_write_end()) page@000000004a97ae42[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.367159:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000cb61bbe2 17ffffc0001001 3:0 ffff8f994f1c29a0 575 lru 00000080:00200000:2.0:1775764854.367164:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000059a93bcd 575: 1< 0 - - > 2< 2355200 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.367166:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000004a97ae42 00000080:00200000:2.0:1775764854.367167:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 64. 00000080:00200000:2.0:1775764854.368059:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 576 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.368097:0:22506:0:(rw26.c:858:ll_write_end()) pos 2359296, len 4096, copied 4096 00000080:00200000:2.0:1775764854.368104:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000199458e6[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.368106:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000b6a951a6 17ffffc0001001 3:0 ffff8f994f1c2000 576 lru 00000080:00200000:2.0:1775764854.368113:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000ccc572f8 576: 1< 0 - - > 2< 2359296 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.368115:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000199458e6 00000080:00200000:2.0:1775764854.368116:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 65. 00000080:00200000:2.0:1775764854.368118:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 577 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.368136:0:22506:0:(rw26.c:858:ll_write_end()) pos 2363392, len 4096, copied 4096 00000080:00200000:2.0:1775764854.368139:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000019c9bd80[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.368142:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000df9168a3 17ffffc0001001 3:0 ffff8f994f1c21c0 577 lru 00000080:00200000:2.0:1775764854.368147:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000045273b76 577: 1< 0 - - > 2< 2363392 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.368148:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000019c9bd80 00000080:00200000:2.0:1775764854.368149:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 66. 00000080:00200000:2.0:1775764854.368151:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 578 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.368166:0:22506:0:(rw26.c:858:ll_write_end()) pos 2367488, len 4096, copied 4096 00000080:00200000:2.0:1775764854.368254:0:22506:0:(rw26.c:916:ll_write_end()) page@000000002c744b51[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.368257:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000005d93e7d8 17ffffc0001001 3:0 ffff8f994f1c2540 578 lru 00000080:00200000:2.0:1775764854.368262:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000c8cdfcac 578: 1< 0 - - > 2< 2367488 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.368263:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000002c744b51 00000080:00200000:2.0:1775764854.368265:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 67. 00000080:00200000:2.0:1775764854.368267:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 579 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.368283:0:22506:0:(rw26.c:858:ll_write_end()) pos 2371584, len 4096, copied 4096 00000080:00200000:2.0:1775764854.368286:0:22506:0:(rw26.c:916:ll_write_end()) page@000000006099e3b8[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.368289:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000c5f741d5 17ffffc0001001 3:0 ffff8f994f1c2460 579 lru 00000080:00200000:2.0:1775764854.368293:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000003110870e 579: 1< 0 - - > 2< 2371584 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.368295:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000006099e3b8 00000080:00200000:2.0:1775764854.368296:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 68. 00000080:00200000:2.0:1775764854.368298:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 580 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.368309:0:22506:0:(rw26.c:858:ll_write_end()) pos 2375680, len 4096, copied 4096 00000080:00200000:2.0:1775764854.368312:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000204fd819[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.368315:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000007c469c31 17ffffc0001001 3:0 ffff8f994f1c2ee0 580 lru 00000080:00200000:2.0:1775764854.368320:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000001122ce7c 580: 1< 0 - - > 2< 2375680 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.368321:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000204fd819 00000080:00200000:2.0:1775764854.368322:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 69. 00000080:00200000:2.0:1775764854.368324:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 581 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.368363:0:22506:0:(rw26.c:858:ll_write_end()) pos 2379776, len 4096, copied 4096 00000080:00200000:2.0:1775764854.368367:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000d226a041[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.368369:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000004dbde8aa 17ffffc0001001 3:0 ffff8f994f20b7e0 581 lru 00000080:00200000:2.0:1775764854.368374:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000047c47fcb 581: 1< 0 - - > 2< 2379776 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.368376:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000d226a041 00000080:00200000:2.0:1775764854.368377:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 70. 00000080:00200000:2.0:1775764854.368379:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 582 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.368398:0:22506:0:(rw26.c:858:ll_write_end()) pos 2383872, len 4096, copied 4096 00000080:00200000:2.0:1775764854.368401:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000a157258d[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.368404:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000c63cac4d 17ffffc0001021 2:0 ffff8f994f20bc40 582 lru 00000080:00200000:2.0:1775764854.368409:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000026e62313 582: 1< 0 - - > 2< 2383872 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.368411:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000a157258d 00000080:00200000:2.0:1775764854.368413:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 71. 00000080:00200000:2.0:1775764854.368414:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 583 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.368424:0:22506:0:(rw26.c:858:ll_write_end()) pos 2387968, len 4096, copied 4096 00000080:00200000:2.0:1775764854.368427:0:22506:0:(rw26.c:916:ll_write_end()) page@000000004d9ffb07[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.368444:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000066602cef 17ffffc0001001 3:0 ffff8f994f20ba80 583 lru 00000080:00200000:2.0:1775764854.368449:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000007d87e28b 583: 1< 0 - - > 2< 2387968 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.368451:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000004d9ffb07 00000080:00200000:2.0:1775764854.368452:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 72. 00000080:00200000:2.0:1775764854.368454:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 584 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.368468:0:22506:0:(rw26.c:858:ll_write_end()) pos 2392064, len 4096, copied 4096 00000080:00200000:2.0:1775764854.368472:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000ded09702[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.368474:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000451f331f 17ffffc0001001 3:0 ffff8f994f20b460 584 lru 00000080:00200000:2.0:1775764854.368480:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000006412bb13 584: 1< 0 - - > 2< 2392064 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.368482:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000ded09702 00000080:00200000:2.0:1775764854.368483:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 73. 00000080:00200000:2.0:1775764854.368485:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 585 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.368496:0:22506:0:(rw26.c:858:ll_write_end()) pos 2396160, len 4096, copied 4096 00000080:00200000:2.0:1775764854.368499:0:22506:0:(rw26.c:916:ll_write_end()) page@000000009d8ff03a[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.368501:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000eca8ea48 17ffffc0001001 3:0 ffff8f994f20b540 585 lru 00000080:00200000:2.0:1775764854.368506:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000002d10ee7f 585: 1< 0 - - > 2< 2396160 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.368508:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000009d8ff03a 00000080:00200000:2.0:1775764854.368509:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 74. 00000080:00200000:2.0:1775764854.368511:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 586 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.368523:0:22506:0:(rw26.c:858:ll_write_end()) pos 2400256, len 4096, copied 4096 00000080:00200000:2.0:1775764854.368526:0:22506:0:(rw26.c:916:ll_write_end()) page@000000003cb1706a[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.368528:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000008949d4e0 17ffffc0001001 3:0 ffff8f994f20b8c0 586 lru 00000080:00200000:2.0:1775764854.368534:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000e2ebe481 586: 1< 0 - - > 2< 2400256 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.368536:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000003cb1706a 00000080:00200000:2.0:1775764854.368537:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 75. 00000080:00200000:2.0:1775764854.368538:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 587 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.368548:0:22506:0:(rw26.c:858:ll_write_end()) pos 2404352, len 4096, copied 4096 00000080:00200000:2.0:1775764854.368551:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000ac2992a5[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.368553:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000009f2a7524 17ffffc0001001 3:0 ffff8f994f20bee0 587 lru 00000080:00200000:2.0:1775764854.368557:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000001b2f3f6a 587: 1< 0 - - > 2< 2404352 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.368559:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000ac2992a5 00000080:00200000:2.0:1775764854.368560:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 76. 00000080:00200000:2.0:1775764854.368562:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 588 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.368572:0:22506:0:(rw26.c:858:ll_write_end()) pos 2408448, len 4096, copied 4096 00000080:00200000:2.0:1775764854.368575:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000bedfe33d[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.368577:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000097a8c80d 17ffffc0001001 3:0 ffff8f994f20b700 588 lru 00000080:00200000:2.0:1775764854.368582:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000051c003c1 588: 1< 0 - - > 2< 2408448 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.368583:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000bedfe33d 00000080:00200000:2.0:1775764854.368584:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 77. 00000080:00200000:2.0:1775764854.368586:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 589 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.368596:0:22506:0:(rw26.c:858:ll_write_end()) pos 2412544, len 4096, copied 4096 00000080:00200000:2.0:1775764854.368598:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000e5ff935a[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.368601:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000000495a9db 17ffffc0001001 3:0 ffff8f994f20bd20 589 lru 00000080:00200000:2.0:1775764854.368605:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000a6b0e508 589: 1< 0 - - > 2< 2412544 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.368607:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000e5ff935a 00000080:00200000:2.0:1775764854.368608:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 78. 00000080:00200000:2.0:1775764854.368610:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 590 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.368705:0:22506:0:(rw26.c:858:ll_write_end()) pos 2416640, len 4096, copied 4096 00000080:00200000:2.0:1775764854.368710:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000087448f4a[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.368713:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000d3e57b1c 17ffffc0001001 3:0 ffff8f994f20b2a0 590 lru 00000080:00200000:2.0:1775764854.368718:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000092525dd0 590: 1< 0 - - > 2< 2416640 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.368719:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000087448f4a 00000080:00200000:2.0:1775764854.368720:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 79. 00000080:00200000:2.0:1775764854.368722:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 591 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.368735:0:22506:0:(rw26.c:858:ll_write_end()) pos 2420736, len 4096, copied 4096 00000080:00200000:2.0:1775764854.368811:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000d2a46c20[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.368814:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000075ceb386 17ffffc0001001 3:0 ffff8f994f20bb60 591 lru 00000080:00200000:2.0:1775764854.368819:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000003ef99c1f 591: 1< 0 - - > 2< 2420736 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.368821:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000d2a46c20 00000080:00200000:2.0:1775764854.368822:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 80. 00000080:00200000:2.0:1775764854.368823:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 592 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.368839:0:22506:0:(rw26.c:858:ll_write_end()) pos 2424832, len 4096, copied 4096 00000080:00200000:2.0:1775764854.368842:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000016957084[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.368855:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000272e0b19 17ffffc0001001 3:0 ffff8f994f20b380 592 lru 00000080:00200000:2.0:1775764854.368860:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000006227ef66 592: 1< 0 - - > 2< 2424832 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.368862:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000016957084 00000080:00200000:2.0:1775764854.368863:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 81. 00000080:00200000:2.0:1775764854.368865:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 593 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.368880:0:22506:0:(rw26.c:858:ll_write_end()) pos 2428928, len 4096, copied 4096 00000080:00200000:2.0:1775764854.368884:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000954f86d7[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.368886:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000005e961c0 17ffffc0001001 3:0 ffff8f994f20b0e0 593 lru 00000080:00200000:2.0:1775764854.368891:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000008fe70202 593: 1< 0 - - > 2< 2428928 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.368892:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000954f86d7 00000080:00200000:2.0:1775764854.368893:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 82. 00000080:00200000:2.0:1775764854.368895:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 594 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.368907:0:22506:0:(rw26.c:858:ll_write_end()) pos 2433024, len 4096, copied 4096 00000080:00200000:2.0:1775764854.368910:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000cfe8b512[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.368912:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000d9a425cd 17ffffc0001001 3:0 ffff8f994f20be00 594 lru 00000080:00200000:2.0:1775764854.368917:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000391d42aa 594: 1< 0 - - > 2< 2433024 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.368918:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000cfe8b512 00000080:00200000:2.0:1775764854.368919:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 83. 00000080:00200000:2.0:1775764854.368921:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 595 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.368932:0:22506:0:(rw26.c:858:ll_write_end()) pos 2437120, len 4096, copied 4096 00000080:00200000:2.0:1775764854.368936:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000455a4a42[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.368938:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000087135b2 17ffffc0001001 3:0 ffff8f994f20b620 595 lru 00000080:00200000:2.0:1775764854.368943:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000003855ac3a 595: 1< 0 - - > 2< 2437120 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.368945:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000455a4a42 00000080:00200000:2.0:1775764854.368946:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 84. 00000080:00200000:2.0:1775764854.368948:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 596 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.368961:0:22506:0:(rw26.c:858:ll_write_end()) pos 2441216, len 4096, copied 4096 00000080:00200000:2.0:1775764854.368964:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000cfa37cde[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.368967:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000698c036d 17ffffc0001001 3:0 ffff8f994f20b9a0 596 lru 00000080:00200000:2.0:1775764854.368972:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000006f675dae 596: 1< 0 - - > 2< 2441216 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.368974:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000cfa37cde 00000080:00200000:2.0:1775764854.368975:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 85. 00000080:00200000:2.0:1775764854.368977:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 597 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.368994:0:22506:0:(rw26.c:858:ll_write_end()) pos 2445312, len 4096, copied 4096 00000080:00200000:2.0:1775764854.368997:0:22506:0:(rw26.c:916:ll_write_end()) page@000000003ffbc7a3[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.368999:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000f2d3b0d4 17ffffc0001021 2:0 ffff8f994f20b1c0 597 lru 00000080:00200000:2.0:1775764854.369004:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000094c22e20 597: 1< 0 - - > 2< 2445312 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.369006:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000003ffbc7a3 00000080:00200000:2.0:1775764854.369007:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 86. 00000080:00200000:2.0:1775764854.369008:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 598 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.369020:0:22506:0:(rw26.c:858:ll_write_end()) pos 2449408, len 4096, copied 4096 00000080:00200000:2.0:1775764854.369023:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000cadf1d50[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.369025:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000003d0fc64f 17ffffc0001001 3:0 ffff8f994f20b000 598 lru 00000080:00200000:2.0:1775764854.369030:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000047219a79 598: 1< 0 - - > 2< 2449408 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.369032:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000cadf1d50 00000080:00200000:2.0:1775764854.369033:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 87. 00000080:00200000:2.0:1775764854.369035:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 599 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.369047:0:22506:0:(rw26.c:858:ll_write_end()) pos 2453504, len 4096, copied 4096 00000080:00200000:2.0:1775764854.369051:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000009f3c97[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.369053:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000008faf8485 17ffffc0001001 3:0 ffff8f99668bb0e0 599 lru 00000080:00200000:2.0:1775764854.369058:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000003eaaab56 599: 1< 0 - - > 2< 2453504 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.369059:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000009f3c97 00000080:00200000:2.0:1775764854.369060:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 88. 00000080:00200000:2.0:1775764854.369062:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 600 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.369075:0:22506:0:(rw26.c:858:ll_write_end()) pos 2457600, len 4096, copied 4096 00000080:00200000:2.0:1775764854.369079:0:22506:0:(rw26.c:916:ll_write_end()) page@000000000d8d2c6a[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.369081:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000985afee1 17ffffc0001001 3:0 ffff8f99668bb2a0 600 lru 00000080:00200000:2.0:1775764854.369086:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000008d4915ec 600: 1< 0 - - > 2< 2457600 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.369087:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000000d8d2c6a 00000080:00200000:2.0:1775764854.369088:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 89. 00000080:00200000:2.0:1775764854.369090:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 601 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.369101:0:22506:0:(rw26.c:858:ll_write_end()) pos 2461696, len 4096, copied 4096 00000080:00200000:2.0:1775764854.369105:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000ee243e3a[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.369107:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000dfd7de21 17ffffc0001001 3:0 ffff8f99668bb380 601 lru 00000080:00200000:2.0:1775764854.369112:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000009172318b 601: 1< 0 - - > 2< 2461696 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.369113:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000ee243e3a 00000080:00200000:2.0:1775764854.369114:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 90. 00000080:00200000:2.0:1775764854.369115:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 602 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.369128:0:22506:0:(rw26.c:858:ll_write_end()) pos 2465792, len 4096, copied 4096 00000080:00200000:2.0:1775764854.369131:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000c715dd82[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.369133:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000073a37401 17ffffc0001001 3:0 ffff8f99668bbb60 602 lru 00000080:00200000:2.0:1775764854.369137:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000003d958f3a 602: 1< 0 - - > 2< 2465792 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.369139:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000c715dd82 00000080:00200000:2.0:1775764854.369140:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 91. 00000080:00200000:2.0:1775764854.369142:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 603 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.369154:0:22506:0:(rw26.c:858:ll_write_end()) pos 2469888, len 4096, copied 4096 00000080:00200000:2.0:1775764854.369157:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000785e3240[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.369160:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000b14e838d 17ffffc0001001 3:0 ffff8f99668bb000 603 lru 00000080:00200000:2.0:1775764854.369165:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000848ff1e2 603: 1< 0 - - > 2< 2469888 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.369167:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000785e3240 00000080:00200000:2.0:1775764854.369168:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 92. 00000080:00200000:2.0:1775764854.369246:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 604 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.369263:0:22506:0:(rw26.c:858:ll_write_end()) pos 2473984, len 4096, copied 4096 00000080:00200000:2.0:1775764854.369267:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000dd6c1b18[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.369270:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000003b68e5c9 17ffffc0001001 3:0 ffff8f99668bb1c0 604 lru 00000080:00200000:2.0:1775764854.369275:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000006116ea73 604: 1< 0 - - > 2< 2473984 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.369277:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000dd6c1b18 00000080:00200000:2.0:1775764854.369278:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 93. 00000080:00200000:2.0:1775764854.369281:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 605 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.369296:0:22506:0:(rw26.c:858:ll_write_end()) pos 2478080, len 4096, copied 4096 00000080:00200000:2.0:1775764854.369300:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000fef03d67[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.369302:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000028f0e8c1 17ffffc0001001 3:0 ffff8f99668bb620 605 lru 00000080:00200000:2.0:1775764854.369308:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000c4820f86 605: 1< 0 - - > 2< 2478080 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.369310:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000fef03d67 00000080:00200000:2.0:1775764854.369311:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 94. 00000080:00200000:2.0:1775764854.369313:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 606 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.369327:0:22506:0:(rw26.c:858:ll_write_end()) pos 2482176, len 4096, copied 4096 00000080:00200000:2.0:1775764854.369330:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000086434053[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.369333:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000085bab6ed 17ffffc0001001 3:0 ffff8f99668bbe00 606 lru 00000080:00200000:2.0:1775764854.369338:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000098124e80 606: 1< 0 - - > 2< 2482176 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.369340:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000086434053 00000080:00200000:2.0:1775764854.369342:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 95. 00000080:00200000:2.0:1775764854.369344:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 607 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.369359:0:22506:0:(rw26.c:858:ll_write_end()) pos 2486272, len 4096, copied 4096 00000080:00200000:2.0:1775764854.369363:0:22506:0:(rw26.c:916:ll_write_end()) page@000000003b296ca6[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.369366:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000040f429c 17ffffc0001001 3:0 ffff8f99668bbc40 607 lru 00000080:00200000:2.0:1775764854.369371:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000ae983e71 607: 1< 0 - - > 2< 2486272 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.369373:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000003b296ca6 00000080:00200000:2.0:1775764854.369374:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 96. 00000080:00200000:2.0:1775764854.369376:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 608 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.369389:0:22506:0:(rw26.c:858:ll_write_end()) pos 2490368, len 4096, copied 4096 00000080:00200000:2.0:1775764854.369392:0:22506:0:(rw26.c:916:ll_write_end()) page@000000003fabb9ec[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.369395:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000090c0e663 17ffffc0001001 3:0 ffff8f99668bb8c0 608 lru 00000080:00200000:2.0:1775764854.369400:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000006a2e7ea4 608: 1< 0 - - > 2< 2490368 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.369402:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000003fabb9ec 00000080:00200000:2.0:1775764854.369403:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 97. 00000080:00200000:2.0:1775764854.369405:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 609 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.369420:0:22506:0:(rw26.c:858:ll_write_end()) pos 2494464, len 4096, copied 4096 00000080:00200000:2.0:1775764854.369423:0:22506:0:(rw26.c:916:ll_write_end()) page@000000004770be71[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.369426:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000f790841e 17ffffc0001001 3:0 ffff8f99668bbd20 609 lru 00000080:00200000:2.0:1775764854.369431:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000a451c2fe 609: 1< 0 - - > 2< 2494464 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.369433:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000004770be71 00000080:00200000:2.0:1775764854.369434:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 98. 00000080:00200000:2.0:1775764854.369436:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 610 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.369450:0:22506:0:(rw26.c:858:ll_write_end()) pos 2498560, len 4096, copied 4096 00000080:00200000:2.0:1775764854.369454:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000037c39742[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.369457:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000032358073 17ffffc0001001 3:0 ffff8f99668bb460 610 lru 00000080:00200000:2.0:1775764854.369462:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000002edae5fb 610: 1< 0 - - > 2< 2498560 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.369464:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000037c39742 00000080:00200000:2.0:1775764854.369464:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 99. 00000080:00200000:2.0:1775764854.369466:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 611 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.369480:0:22506:0:(rw26.c:858:ll_write_end()) pos 2502656, len 4096, copied 4096 00000080:00200000:2.0:1775764854.369484:0:22506:0:(rw26.c:916:ll_write_end()) page@000000001c3f0b92[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.369487:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000d064329e 17ffffc0001001 3:0 ffff8f99668bb700 611 lru 00000080:00200000:2.0:1775764854.369492:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000ba910931 611: 1< 0 - - > 2< 2502656 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.369494:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000001c3f0b92 00000080:00200000:2.0:1775764854.369495:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 100. 00000080:00200000:2.0:1775764854.369497:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 612 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.369515:0:22506:0:(rw26.c:858:ll_write_end()) pos 2506752, len 4096, copied 4096 00000080:00200000:2.0:1775764854.369519:0:22506:0:(rw26.c:916:ll_write_end()) page@000000004102ca4c[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.369522:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000c113a4fa 17ffffc0001021 2:0 ffff8f99668bb7e0 612 lru 00000080:00200000:2.0:1775764854.369528:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000002974ad93 612: 1< 0 - - > 2< 2506752 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.369529:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000004102ca4c 00000080:00200000:2.0:1775764854.369530:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 101. 00000080:00200000:2.0:1775764854.369532:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 613 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.369546:0:22506:0:(rw26.c:858:ll_write_end()) pos 2510848, len 4096, copied 4096 00000080:00200000:2.0:1775764854.369550:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000022e33e50[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.369553:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000f10c0c7a 17ffffc0001001 3:0 ffff8f99668bb9a0 613 lru 00000080:00200000:2.0:1775764854.369558:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000c97ef24b 613: 1< 0 - - > 2< 2510848 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.369560:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000022e33e50 00000080:00200000:2.0:1775764854.369561:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 102. 00000080:00200000:2.0:1775764854.369563:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 614 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.369578:0:22506:0:(rw26.c:858:ll_write_end()) pos 2514944, len 4096, copied 4096 00000080:00200000:2.0:1775764854.369582:0:22506:0:(rw26.c:916:ll_write_end()) page@000000004634d642[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.369584:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000f7f7a2cc 17ffffc0001001 3:0 ffff8f99668bb540 614 lru 00000080:00200000:2.0:1775764854.369589:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000004e96bbfe 614: 1< 0 - - > 2< 2514944 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.369590:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000004634d642 00000080:00200000:2.0:1775764854.369591:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 103. 00000080:00200000:2.0:1775764854.369593:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 615 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.369606:0:22506:0:(rw26.c:858:ll_write_end()) pos 2519040, len 4096, copied 4096 00000080:00200000:2.0:1775764854.369609:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000034b17559[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.369712:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000627cc9bd 17ffffc0001001 3:0 ffff8f99668bbee0 615 lru 00000080:00200000:2.0:1775764854.369718:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000db7de765 615: 1< 0 - - > 2< 2519040 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.369719:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000034b17559 00000080:00200000:2.0:1775764854.369721:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 104. 00000080:00200000:2.0:1775764854.369723:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 616 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.369820:0:22506:0:(rw26.c:858:ll_write_end()) pos 2523136, len 4096, copied 4096 00000080:00200000:2.0:1775764854.369825:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000002a907a[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.369827:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000098beedc0 17ffffc0001001 3:0 ffff8f99668bba80 616 lru 00000080:00200000:2.0:1775764854.369832:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000003b58956b 616: 1< 0 - - > 2< 2523136 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.369834:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000002a907a 00000080:00200000:2.0:1775764854.369835:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 105. 00000080:00200000:2.0:1775764854.369837:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 617 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.369864:0:22506:0:(rw26.c:858:ll_write_end()) pos 2527232, len 4096, copied 4096 00000080:00200000:2.0:1775764854.369868:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000221dd1c6[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.369871:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000a6f7bbb0 17ffffc0001001 3:0 ffff8f99669447e0 617 lru 00000080:00200000:2.0:1775764854.369876:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000f79202cb 617: 1< 0 - - > 2< 2527232 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.369878:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000221dd1c6 00000080:00200000:2.0:1775764854.369879:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 106. 00000080:00200000:2.0:1775764854.369880:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 618 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.369895:0:22506:0:(rw26.c:858:ll_write_end()) pos 2531328, len 4096, copied 4096 00000080:00200000:2.0:1775764854.369900:0:22506:0:(rw26.c:916:ll_write_end()) page@000000007d5b29a4[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.369902:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000004f7bb36c 17ffffc0001001 3:0 ffff8f99669449a0 618 lru 00000080:00200000:2.0:1775764854.369907:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000001166e183 618: 1< 0 - - > 2< 2531328 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.369909:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000007d5b29a4 00000080:00200000:2.0:1775764854.369910:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 107. 00000080:00200000:2.0:1775764854.369912:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 619 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.369924:0:22506:0:(rw26.c:858:ll_write_end()) pos 2535424, len 4096, copied 4096 00000080:00200000:2.0:1775764854.369927:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000067bf8d66[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.369930:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000002799bb6 17ffffc0001001 3:0 ffff8f9966944ee0 619 lru 00000080:00200000:2.0:1775764854.369935:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000f4921a1e 619: 1< 0 - - > 2< 2535424 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.369937:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000067bf8d66 00000080:00200000:2.0:1775764854.369938:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 108. 00000080:00200000:2.0:1775764854.369940:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 620 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.369954:0:22506:0:(rw26.c:858:ll_write_end()) pos 2539520, len 4096, copied 4096 00000080:00200000:2.0:1775764854.369958:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000022b725c1[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.369960:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000a377aef8 17ffffc0001001 3:0 ffff8f9966944a80 620 lru 00000080:00200000:2.0:1775764854.369964:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000004bdfe50c 620: 1< 0 - - > 2< 2539520 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.369966:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000022b725c1 00000080:00200000:2.0:1775764854.369967:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 109. 00000080:00200000:2.0:1775764854.369969:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 621 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.369982:0:22506:0:(rw26.c:858:ll_write_end()) pos 2543616, len 4096, copied 4096 00000080:00200000:2.0:1775764854.369985:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000d48e97e2[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.369988:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000061313504 17ffffc0001001 3:0 ffff8f9966944540 621 lru 00000080:00200000:2.0:1775764854.369993:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000bd96bbfa 621: 1< 0 - - > 2< 2543616 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.369995:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000d48e97e2 00000080:00200000:2.0:1775764854.369996:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 110. 00000080:00200000:2.0:1775764854.369997:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 622 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.370007:0:22506:0:(rw26.c:858:ll_write_end()) pos 2547712, len 4096, copied 4096 00000080:00200000:2.0:1775764854.370010:0:22506:0:(rw26.c:916:ll_write_end()) page@000000007d934cd7[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.370013:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000008ed190ae 17ffffc0001001 3:0 ffff8f9966944460 622 lru 00000080:00200000:2.0:1775764854.370018:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000007cc88a8 622: 1< 0 - - > 2< 2547712 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.370020:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000007d934cd7 00000080:00200000:2.0:1775764854.370021:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 111. 00000080:00200000:2.0:1775764854.370022:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 623 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.370035:0:22506:0:(rw26.c:858:ll_write_end()) pos 2551808, len 4096, copied 4096 00000080:00200000:2.0:1775764854.370038:0:22506:0:(rw26.c:916:ll_write_end()) page@000000000c63185c[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.370041:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000a5b3de69 17ffffc0001001 3:0 ffff8f9966944000 623 lru 00000080:00200000:2.0:1775764854.370046:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000001db0c519 623: 1< 0 - - > 2< 2551808 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.370048:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000000c63185c 00000080:00200000:2.0:1775764854.370050:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 112. 00000080:00200000:2.0:1775764854.370051:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 624 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.370062:0:22506:0:(rw26.c:858:ll_write_end()) pos 2555904, len 4096, copied 4096 00000080:00200000:2.0:1775764854.370065:0:22506:0:(rw26.c:916:ll_write_end()) page@000000004367ebd5[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.370067:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000006c6f514d 17ffffc0001001 3:0 ffff8f99669441c0 624 lru 00000080:00200000:2.0:1775764854.370073:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000003a835c01 624: 1< 0 - - > 2< 2555904 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.370074:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000004367ebd5 00000080:00200000:2.0:1775764854.370075:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 113. 00000080:00200000:2.0:1775764854.370077:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 625 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.370091:0:22506:0:(rw26.c:858:ll_write_end()) pos 2560000, len 4096, copied 4096 00000080:00200000:2.0:1775764854.370095:0:22506:0:(rw26.c:916:ll_write_end()) page@000000000ede63c6[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.370097:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000d37b117a 17ffffc0001001 3:0 ffff8f9966944620 625 lru 00000080:00200000:2.0:1775764854.370101:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000ff1c5202 625: 1< 0 - - > 2< 2560000 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.370103:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000000ede63c6 00000080:00200000:2.0:1775764854.370104:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 114. 00000080:00200000:2.0:1775764854.370105:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 626 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.370118:0:22506:0:(rw26.c:858:ll_write_end()) pos 2564096, len 4096, copied 4096 00000080:00200000:2.0:1775764854.370121:0:22506:0:(rw26.c:916:ll_write_end()) page@000000000666b9e3[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.370123:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000001d0c8fb9 17ffffc0001001 3:0 ffff8f9966944e00 626 lru 00000080:00200000:2.0:1775764854.370128:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000dd99ca21 626: 1< 0 - - > 2< 2564096 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.370130:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000000666b9e3 00000080:00200000:2.0:1775764854.370131:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 115. 00000080:00200000:2.0:1775764854.370133:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 627 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.370151:0:22506:0:(rw26.c:858:ll_write_end()) pos 2568192, len 4096, copied 4096 00000080:00200000:2.0:1775764854.370154:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000c5ae67ae[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.370157:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000001516b101 17ffffc0001021 2:0 ffff8f9966944c40 627 lru 00000080:00200000:2.0:1775764854.370162:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000000773c528 627: 1< 0 - - > 2< 2568192 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.370163:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000c5ae67ae 00000080:00200000:2.0:1775764854.370164:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 116. 00000080:00200000:2.0:1775764854.370166:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 628 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.370255:0:22506:0:(rw26.c:858:ll_write_end()) pos 2572288, len 4096, copied 4096 00000080:00200000:2.0:1775764854.370259:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000077c6e4e7[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.370262:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000dad2abf4 17ffffc0001001 3:0 ffff8f99669448c0 628 lru 00000080:00200000:2.0:1775764854.370267:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000006d9f58dd 628: 1< 0 - - > 2< 2572288 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.370269:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000077c6e4e7 00000080:00200000:2.0:1775764854.370270:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 117. 00000080:00200000:2.0:1775764854.370272:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 629 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.370282:0:22506:0:(rw26.c:858:ll_write_end()) pos 2576384, len 4096, copied 4096 00000080:00200000:2.0:1775764854.370285:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000395d5f6f[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.370288:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000008fb280de 17ffffc0001001 3:0 ffff8f9966944d20 629 lru 00000080:00200000:2.0:1775764854.370293:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000004a4fb066 629: 1< 0 - - > 2< 2576384 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.370295:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000395d5f6f 00000080:00200000:2.0:1775764854.370296:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 118. 00000080:00200000:2.0:1775764854.370297:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 630 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.370307:0:22506:0:(rw26.c:858:ll_write_end()) pos 2580480, len 4096, copied 4096 00000080:00200000:2.0:1775764854.370310:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000271fa017[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.370312:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000fb2e7fa8 17ffffc0001001 3:0 ffff8f99669440e0 630 lru 00000080:00200000:2.0:1775764854.370317:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000b7172633 630: 1< 0 - - > 2< 2580480 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.370319:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000271fa017 00000080:00200000:2.0:1775764854.370320:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 119. 00000080:00200000:2.0:1775764854.370321:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 631 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.370332:0:22506:0:(rw26.c:858:ll_write_end()) pos 2584576, len 4096, copied 4096 00000080:00200000:2.0:1775764854.370335:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000054712e3e[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.370338:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000bbf28d1a 17ffffc0001001 3:0 ffff8f99669442a0 631 lru 00000080:00200000:2.0:1775764854.370343:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000089c5383c 631: 1< 0 - - > 2< 2584576 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.370345:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000054712e3e 00000080:00200000:2.0:1775764854.370346:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 120. 00000080:00200000:2.0:1775764854.370348:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 632 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.370358:0:22506:0:(rw26.c:858:ll_write_end()) pos 2588672, len 4096, copied 4096 00000080:00200000:2.0:1775764854.370361:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000a4e368d0[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.370364:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000ff665ce2 17ffffc0001001 3:0 ffff8f9966944380 632 lru 00000080:00200000:2.0:1775764854.370368:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000000108ce1c 632: 1< 0 - - > 2< 2588672 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.370370:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000a4e368d0 00000080:00200000:2.0:1775764854.370372:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 121. 00000080:00200000:2.0:1775764854.370373:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 633 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.370382:0:22506:0:(rw26.c:858:ll_write_end()) pos 2592768, len 4096, copied 4096 00000080:00200000:2.0:1775764854.370385:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000043f9b5e3[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.370388:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000001bdccabc 17ffffc0001001 3:0 ffff8f9966944b60 633 lru 00000080:00200000:2.0:1775764854.370393:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000008ccd58ce 633: 1< 0 - - > 2< 2592768 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.370395:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000043f9b5e3 00000080:00200000:2.0:1775764854.370396:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 122. 00000080:00200000:2.0:1775764854.370398:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 634 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.370408:0:22506:0:(rw26.c:858:ll_write_end()) pos 2596864, len 4096, copied 4096 00000080:00200000:2.0:1775764854.370411:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000d9cc7004[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.370414:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000077a9ed17 17ffffc0001001 3:0 ffff8f9966944700 634 lru 00000080:00200000:2.0:1775764854.370419:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000003f486602 634: 1< 0 - - > 2< 2596864 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.370421:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000d9cc7004 00000080:00200000:2.0:1775764854.370422:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 123. 00000080:00200000:2.0:1775764854.370424:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 635 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.370462:0:22506:0:(rw26.c:858:ll_write_end()) pos 2600960, len 4096, copied 4096 00000080:00200000:2.0:1775764854.370466:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000fb4e1723[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.370469:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000003cd6035e 17ffffc0001001 3:0 ffff8f994f20fb60 635 lru 00000080:00200000:2.0:1775764854.370474:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000019f536ef 635: 1< 0 - - > 2< 2600960 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.370476:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000fb4e1723 00000080:00200000:2.0:1775764854.370477:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 124. 00000080:00200000:2.0:1775764854.370479:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 636 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.370491:0:22506:0:(rw26.c:858:ll_write_end()) pos 2605056, len 4096, copied 4096 00000080:00200000:2.0:1775764854.370494:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000090d1e40b[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.370496:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000007e62a823 17ffffc0001001 3:0 ffff8f994f20f700 636 lru 00000080:00200000:2.0:1775764854.370501:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000062d5866f 636: 1< 0 - - > 2< 2605056 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.370503:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000090d1e40b 00000080:00200000:2.0:1775764854.370504:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 125. 00000080:00200000:2.0:1775764854.370506:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 637 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.370515:0:22506:0:(rw26.c:858:ll_write_end()) pos 2609152, len 4096, copied 4096 00000080:00200000:2.0:1775764854.370518:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000333a468c[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.370521:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000006bed9e7e 17ffffc0001001 3:0 ffff8f994f20f7e0 637 lru 00000080:00200000:2.0:1775764854.370525:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000002fdeb8ac 637: 1< 0 - - > 2< 2609152 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.370527:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000333a468c 00000080:00200000:2.0:1775764854.370528:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 126. 00000080:00200000:2.0:1775764854.370529:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 638 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.370540:0:22506:0:(rw26.c:858:ll_write_end()) pos 2613248, len 4096, copied 4096 00000080:00200000:2.0:1775764854.370543:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000511ec51b[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.370545:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000007198d40b 17ffffc0001001 3:0 ffff8f994f20f9a0 638 lru 00000080:00200000:2.0:1775764854.370549:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000002f94aeea 638: 1< 0 - - > 2< 2613248 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.370551:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000511ec51b 00000080:00200000:2.0:1775764854.370552:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 127. 00000080:00200000:2.0:1775764854.370553:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 639 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.370564:0:22506:0:(rw26.c:858:ll_write_end()) pos 2617344, len 4096, copied 4096 00000080:00200000:2.0:1775764854.370567:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000247ea3dc[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.370570:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000d3ce2a31 17ffffc0001001 3:0 ffff8f994f20fee0 639 lru 00000080:00200000:2.0:1775764854.370575:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000029ce48db 639: 1< 0 - - > 2< 2617344 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.370577:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000247ea3dc 00000080:00200000:2.0:1775764854.370578:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 128. 00000080:00200000:2.0:1775764854.370580:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 640 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.370596:0:22506:0:(rw26.c:858:ll_write_end()) pos 2621440, len 4096, copied 4096 00000080:00200000:2.0:1775764854.370599:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000bc02a334[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.370602:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000007a493e7a 17ffffc0001001 3:0 ffff8f994f20fa80 640 lru 00000080:00200000:2.0:1775764854.370607:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000015e66759 640: 1< 0 - - > 2< 2621440 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.370609:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000bc02a334 00000080:00200000:2.0:1775764854.370706:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 129. 00000080:00200000:2.0:1775764854.370709:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 641 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.370727:0:22506:0:(rw26.c:858:ll_write_end()) pos 2625536, len 4096, copied 4096 00000080:00200000:2.0:1775764854.370730:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000000dc066c[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.370733:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000dc784f6a 17ffffc0001001 3:0 ffff8f994f20f540 641 lru 00000080:00200000:2.0:1775764854.370822:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000047300ce3 641: 1< 0 - - > 2< 2625536 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.370825:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000000dc066c 00000080:00200000:2.0:1775764854.370826:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 130. 00000080:00200000:2.0:1775764854.370828:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 642 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.370860:0:22506:0:(rw26.c:858:ll_write_end()) pos 2629632, len 4096, copied 4096 00000080:00200000:2.0:1775764854.370864:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000ea7793f2[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.370866:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000090f09834 17ffffc0001021 2:0 ffff8f994f20f460 642 lru 00000080:00200000:2.0:1775764854.370870:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000000169ae3e 642: 1< 0 - - > 2< 2629632 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.370872:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000ea7793f2 00000080:00200000:2.0:1775764854.370873:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 131. 00000080:00200000:2.0:1775764854.370875:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 643 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.370888:0:22506:0:(rw26.c:858:ll_write_end()) pos 2633728, len 4096, copied 4096 00000080:00200000:2.0:1775764854.370892:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000f9019597[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.370894:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000b4794986 17ffffc0001001 3:0 ffff8f994f20f000 643 lru 00000080:00200000:2.0:1775764854.370899:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000008019d1b8 643: 1< 0 - - > 2< 2633728 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.370901:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000f9019597 00000080:00200000:2.0:1775764854.370902:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 132. 00000080:00200000:2.0:1775764854.370904:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 644 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.370917:0:22506:0:(rw26.c:858:ll_write_end()) pos 2637824, len 4096, copied 4096 00000080:00200000:2.0:1775764854.370921:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000072140df[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.370923:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000007510f4d8 17ffffc0001001 3:0 ffff8f994f20f1c0 644 lru 00000080:00200000:2.0:1775764854.370927:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000922bc4bf 644: 1< 0 - - > 2< 2637824 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.370929:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000072140df 00000080:00200000:2.0:1775764854.370931:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 133. 00000080:00200000:2.0:1775764854.370932:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 645 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.370944:0:22506:0:(rw26.c:858:ll_write_end()) pos 2641920, len 4096, copied 4096 00000080:00200000:2.0:1775764854.370947:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000a4815e1c[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.370950:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000bccc8a7f 17ffffc0001001 3:0 ffff8f994f20f620 645 lru 00000080:00200000:2.0:1775764854.370954:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000030bcb5ab 645: 1< 0 - - > 2< 2641920 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.370956:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000a4815e1c 00000080:00200000:2.0:1775764854.370957:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 134. 00000080:00200000:2.0:1775764854.370958:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 646 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.370970:0:22506:0:(rw26.c:858:ll_write_end()) pos 2646016, len 4096, copied 4096 00000080:00200000:2.0:1775764854.370973:0:22506:0:(rw26.c:916:ll_write_end()) page@000000007b9ae25b[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.370976:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000061048280 17ffffc0001001 3:0 ffff8f994f20fe00 646 lru 00000080:00200000:2.0:1775764854.370981:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000002de48655 646: 1< 0 - - > 2< 2646016 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.370983:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000007b9ae25b 00000080:00200000:2.0:1775764854.370984:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 135. 00000080:00200000:2.0:1775764854.370985:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 647 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.370997:0:22506:0:(rw26.c:858:ll_write_end()) pos 2650112, len 4096, copied 4096 00000080:00200000:2.0:1775764854.371000:0:22506:0:(rw26.c:916:ll_write_end()) page@000000003e964471[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.371002:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000e37d0853 17ffffc0001001 3:0 ffff8f994f20fc40 647 lru 00000080:00200000:2.0:1775764854.371006:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000c570d164 647: 1< 0 - - > 2< 2650112 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.371008:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000003e964471 00000080:00200000:2.0:1775764854.371009:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 136. 00000080:00200000:2.0:1775764854.371011:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 648 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.371022:0:22506:0:(rw26.c:858:ll_write_end()) pos 2654208, len 4096, copied 4096 00000080:00200000:2.0:1775764854.371024:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000f65571d1[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.371027:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000000663d365 17ffffc0001001 3:0 ffff8f994f20f8c0 648 lru 00000080:00200000:2.0:1775764854.371031:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000e42497db 648: 1< 0 - - > 2< 2654208 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.371033:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000f65571d1 00000080:00200000:2.0:1775764854.371034:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 137. 00000080:00200000:2.0:1775764854.371036:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 649 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.371049:0:22506:0:(rw26.c:858:ll_write_end()) pos 2658304, len 4096, copied 4096 00000080:00200000:2.0:1775764854.371052:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000051da8d20[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.371055:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000002f1e2808 17ffffc0001001 3:0 ffff8f994f20fd20 649 lru 00000080:00200000:2.0:1775764854.371060:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000567c6b1c 649: 1< 0 - - > 2< 2658304 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.371062:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000051da8d20 00000080:00200000:2.0:1775764854.371063:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 138. 00000080:00200000:2.0:1775764854.371065:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 650 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.371079:0:22506:0:(rw26.c:858:ll_write_end()) pos 2662400, len 4096, copied 4096 00000080:00200000:2.0:1775764854.371082:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000cb5722c7[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.371084:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000db170fd8 17ffffc0001001 3:0 ffff8f994f20f0e0 650 lru 00000080:00200000:2.0:1775764854.371089:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000ebaf853d 650: 1< 0 - - > 2< 2662400 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.371090:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000cb5722c7 00000080:00200000:2.0:1775764854.371092:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 139. 00000080:00200000:2.0:1775764854.371093:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 651 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.371105:0:22506:0:(rw26.c:858:ll_write_end()) pos 2666496, len 4096, copied 4096 00000080:00200000:2.0:1775764854.371109:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000e5361a1d[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.371111:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000092920c87 17ffffc0001001 3:0 ffff8f994f20f2a0 651 lru 00000080:00200000:2.0:1775764854.371116:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000044efd146 651: 1< 0 - - > 2< 2666496 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.371118:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000e5361a1d 00000080:00200000:2.0:1775764854.371119:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 140. 00000080:00200000:2.0:1775764854.371121:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 652 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.371136:0:22506:0:(rw26.c:858:ll_write_end()) pos 2670592, len 4096, copied 4096 00000080:00200000:2.0:1775764854.371139:0:22506:0:(rw26.c:916:ll_write_end()) page@000000006df8b062[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.371141:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000019b85d39 17ffffc0001001 3:0 ffff8f994f20f380 652 lru 00000080:00200000:2.0:1775764854.371146:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000065443d1 652: 1< 0 - - > 2< 2670592 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.371148:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000006df8b062 00000080:00200000:2.0:1775764854.371150:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 141. 00000080:00200000:2.0:1775764854.371151:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 653 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.371262:0:22506:0:(rw26.c:858:ll_write_end()) pos 2674688, len 4096, copied 4096 00000080:00200000:2.0:1775764854.371266:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000b809e1e6[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.371269:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000006334bfba 17ffffc0001001 3:0 ffff8f9971106000 653 lru 00000080:00200000:2.0:1775764854.371274:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000004d956c3f 653: 1< 0 - - > 2< 2674688 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.371276:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000b809e1e6 00000080:00200000:2.0:1775764854.371277:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 142. 00000080:00200000:2.0:1775764854.371280:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 654 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.371297:0:22506:0:(rw26.c:858:ll_write_end()) pos 2678784, len 4096, copied 4096 00000080:00200000:2.0:1775764854.371301:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000086c5bfd4[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.371304:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000001daf2e00 17ffffc0001001 3:0 ffff8f99711061c0 654 lru 00000080:00200000:2.0:1775764854.371308:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000eaddde74 654: 1< 0 - - > 2< 2678784 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.371310:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000086c5bfd4 00000080:00200000:2.0:1775764854.371311:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 143. 00000080:00200000:2.0:1775764854.371313:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 655 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.371325:0:22506:0:(rw26.c:858:ll_write_end()) pos 2682880, len 4096, copied 4096 00000080:00200000:2.0:1775764854.371329:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000ae2fd97d[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.371332:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000ad01b1ae 17ffffc0001001 3:0 ffff8f9971106620 655 lru 00000080:00200000:2.0:1775764854.371338:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000b855f843 655: 1< 0 - - > 2< 2682880 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.371340:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000ae2fd97d 00000080:00200000:2.0:1775764854.371341:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 144. 00000080:00200000:2.0:1775764854.371343:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 656 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.371354:0:22506:0:(rw26.c:858:ll_write_end()) pos 2686976, len 4096, copied 4096 00000080:00200000:2.0:1775764854.371358:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000085a7389[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.371360:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000b0856e46 17ffffc0001001 3:0 ffff8f9971106e00 656 lru 00000080:00200000:2.0:1775764854.371365:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000002a160ecb 656: 1< 0 - - > 2< 2686976 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.371367:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000085a7389 00000080:00200000:2.0:1775764854.371368:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 145. 00000080:00200000:2.0:1775764854.371370:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 657 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.371385:0:22506:0:(rw26.c:858:ll_write_end()) pos 2691072, len 4096, copied 4096 00000080:00200000:2.0:1775764854.371389:0:22506:0:(rw26.c:916:ll_write_end()) page@000000000fac2e7f[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.371392:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000254691ce 17ffffc0001021 2:0 ffff8f9971106c40 657 lru 00000080:00200000:2.0:1775764854.371397:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000005b5b0fe7 657: 1< 0 - - > 2< 2691072 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.371399:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000000fac2e7f 00000080:00200000:2.0:1775764854.371400:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 146. 00000080:00200000:2.0:1775764854.371402:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 658 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.371416:0:22506:0:(rw26.c:858:ll_write_end()) pos 2695168, len 4096, copied 4096 00000080:00200000:2.0:1775764854.371420:0:22506:0:(rw26.c:916:ll_write_end()) page@000000000e947beb[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.371422:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000005f7b43da 17ffffc0001001 3:0 ffff8f99711068c0 658 lru 00000080:00200000:2.0:1775764854.371427:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000b4b7b021 658: 1< 0 - - > 2< 2695168 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.371429:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000000e947beb 00000080:00200000:2.0:1775764854.371430:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 147. 00000080:00200000:2.0:1775764854.371432:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 659 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.371445:0:22506:0:(rw26.c:858:ll_write_end()) pos 2699264, len 4096, copied 4096 00000080:00200000:2.0:1775764854.371449:0:22506:0:(rw26.c:916:ll_write_end()) page@000000008f5aad9a[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.371451:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000038961a62 17ffffc0001001 3:0 ffff8f9971106d20 659 lru 00000080:00200000:2.0:1775764854.371456:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000e8d7c380 659: 1< 0 - - > 2< 2699264 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.371458:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000008f5aad9a 00000080:00200000:2.0:1775764854.371459:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 148. 00000080:00200000:2.0:1775764854.371461:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 660 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.371475:0:22506:0:(rw26.c:858:ll_write_end()) pos 2703360, len 4096, copied 4096 00000080:00200000:2.0:1775764854.371479:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000462e1932[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.371481:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000f6153fb8 17ffffc0001001 3:0 ffff8f99711060e0 660 lru 00000080:00200000:2.0:1775764854.371486:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000be209049 660: 1< 0 - - > 2< 2703360 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.371489:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000462e1932 00000080:00200000:2.0:1775764854.371490:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 149. 00000080:00200000:2.0:1775764854.371491:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 661 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.371505:0:22506:0:(rw26.c:858:ll_write_end()) pos 2707456, len 4096, copied 4096 00000080:00200000:2.0:1775764854.371509:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000a9da89bc[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.371511:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000c947bf9d 17ffffc0001001 3:0 ffff8f99711062a0 661 lru 00000080:00200000:2.0:1775764854.371515:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000f17bbc97 661: 1< 0 - - > 2< 2707456 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.371517:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000a9da89bc 00000080:00200000:2.0:1775764854.371518:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 150. 00000080:00200000:2.0:1775764854.371520:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 662 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.371532:0:22506:0:(rw26.c:858:ll_write_end()) pos 2711552, len 4096, copied 4096 00000080:00200000:2.0:1775764854.371536:0:22506:0:(rw26.c:916:ll_write_end()) page@000000006a3fc46b[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.371538:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000001aad352 17ffffc0001001 3:0 ffff8f9971106380 662 lru 00000080:00200000:2.0:1775764854.371543:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000c2725ccb 662: 1< 0 - - > 2< 2711552 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.371544:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000006a3fc46b 00000080:00200000:2.0:1775764854.371545:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 151. 00000080:00200000:2.0:1775764854.371547:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 663 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.371562:0:22506:0:(rw26.c:858:ll_write_end()) pos 2715648, len 4096, copied 4096 00000080:00200000:2.0:1775764854.371566:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000d4f8c504[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.371569:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000007c4ca8f2 17ffffc0001001 3:0 ffff8f9971106b60 663 lru 00000080:00200000:2.0:1775764854.371573:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000003d63ae80 663: 1< 0 - - > 2< 2715648 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.371575:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000d4f8c504 00000080:00200000:2.0:1775764854.371576:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 152. 00000080:00200000:2.0:1775764854.371578:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 664 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.371590:0:22506:0:(rw26.c:858:ll_write_end()) pos 2719744, len 4096, copied 4096 00000080:00200000:2.0:1775764854.371593:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000098f28b07[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.371596:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000082a828b0 17ffffc0001001 3:0 ffff8f9971106700 664 lru 00000080:00200000:2.0:1775764854.371601:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000f14c69fe 664: 1< 0 - - > 2< 2719744 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.371603:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000098f28b07 00000080:00200000:2.0:1775764854.371604:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 153. 00000080:00200000:2.0:1775764854.371605:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 665 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.371910:0:22506:0:(rw26.c:858:ll_write_end()) pos 2723840, len 4096, copied 4096 00000080:00200000:2.0:1775764854.371915:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000005556656[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.371917:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000009d9567 17ffffc0001001 3:0 ffff8f99711067e0 665 lru 00000080:00200000:2.0:1775764854.371923:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000ef978c9a 665: 1< 0 - - > 2< 2723840 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.371925:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000005556656 00000080:00200000:2.0:1775764854.371926:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 154. 00000080:00200000:2.0:1775764854.371928:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 666 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.371942:0:22506:0:(rw26.c:858:ll_write_end()) pos 2727936, len 4096, copied 4096 00000080:00200000:2.0:1775764854.371946:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000ac4c268b[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.371948:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000007ffa5045 17ffffc0001001 3:0 ffff8f99711069a0 666 lru 00000080:00200000:2.0:1775764854.371954:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000d2dd6065 666: 1< 0 - - > 2< 2727936 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.371956:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000ac4c268b 00000080:00200000:2.0:1775764854.371957:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 155. 00000080:00200000:2.0:1775764854.371959:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 667 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.371975:0:22506:0:(rw26.c:858:ll_write_end()) pos 2732032, len 4096, copied 4096 00000080:00200000:2.0:1775764854.371979:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000fb3ca725[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.371981:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000066dfed62 17ffffc0001001 3:0 ffff8f9971106ee0 667 lru 00000080:00200000:2.0:1775764854.371986:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000003fc7c3ae 667: 1< 0 - - > 2< 2732032 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.371988:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000fb3ca725 00000080:00200000:2.0:1775764854.371989:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 156. 00000080:00200000:2.0:1775764854.371991:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 668 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.372008:0:22506:0:(rw26.c:858:ll_write_end()) pos 2736128, len 4096, copied 4096 00000080:00200000:2.0:1775764854.372012:0:22506:0:(rw26.c:916:ll_write_end()) page@000000003a66352a[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.372014:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000002280b17e 17ffffc0001001 3:0 ffff8f9971106a80 668 lru 00000080:00200000:2.0:1775764854.372020:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000097da1107 668: 1< 0 - - > 2< 2736128 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.372021:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000003a66352a 00000080:00200000:2.0:1775764854.372023:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 157. 00000080:00200000:2.0:1775764854.372024:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 669 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.372037:0:22506:0:(rw26.c:858:ll_write_end()) pos 2740224, len 4096, copied 4096 00000080:00200000:2.0:1775764854.372041:0:22506:0:(rw26.c:916:ll_write_end()) page@000000005c0812b2[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.372044:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000007d339d76 17ffffc0001001 3:0 ffff8f9971106540 669 lru 00000080:00200000:2.0:1775764854.372049:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000007fe5642 669: 1< 0 - - > 2< 2740224 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.372051:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000005c0812b2 00000080:00200000:2.0:1775764854.372052:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 158. 00000080:00200000:2.0:1775764854.372054:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 670 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.372072:0:22506:0:(rw26.c:858:ll_write_end()) pos 2744320, len 4096, copied 4096 00000080:00200000:2.0:1775764854.372076:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000005a0335c[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.372079:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000325edfa7 17ffffc0001001 3:0 ffff8f9971106460 670 lru 00000080:00200000:2.0:1775764854.372084:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000058cb5a98 670: 1< 0 - - > 2< 2744320 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.372086:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000005a0335c 00000080:00200000:2.0:1775764854.372087:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 159. 00000080:00200000:2.0:1775764854.372089:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 671 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.372125:0:22506:0:(rw26.c:858:ll_write_end()) pos 2748416, len 4096, copied 4096 00000080:00200000:2.0:1775764854.372129:0:22506:0:(rw26.c:916:ll_write_end()) page@000000008c91b535[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.372131:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000004a806509 17ffffc0001001 3:0 ffff8f994f20e2a0 671 lru 00000080:00200000:2.0:1775764854.372136:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000cc54bbc3 671: 1< 0 - - > 2< 2748416 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.372138:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000008c91b535 00000080:00200000:2.0:1775764854.372139:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 160. 00000080:00200000:2.0:1775764854.372141:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 672 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.372157:0:22506:0:(rw26.c:858:ll_write_end()) pos 2752512, len 4096, copied 4096 00000080:00200000:2.0:1775764854.372160:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000f345eb2b[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.372163:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000044cfce5d 17ffffc0001021 2:0 ffff8f994f20e380 672 lru 00000080:00200000:2.0:1775764854.372167:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000c768b5bb 672: 1< 0 - - > 2< 2752512 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.372252:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000f345eb2b 00000080:00200000:2.0:1775764854.372254:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 161. 00000080:00200000:2.0:1775764854.372256:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 673 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.372271:0:22506:0:(rw26.c:858:ll_write_end()) pos 2756608, len 4096, copied 4096 00000080:00200000:2.0:1775764854.372274:0:22506:0:(rw26.c:916:ll_write_end()) page@000000006d5a5319[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.372277:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000f93ac38f 17ffffc0001001 3:0 ffff8f994f20eb60 673 lru 00000080:00200000:2.0:1775764854.372283:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000ccb162a5 673: 1< 0 - - > 2< 2756608 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.372285:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000006d5a5319 00000080:00200000:2.0:1775764854.372286:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 162. 00000080:00200000:2.0:1775764854.372288:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 674 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.372300:0:22506:0:(rw26.c:858:ll_write_end()) pos 2760704, len 4096, copied 4096 00000080:00200000:2.0:1775764854.372304:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000092eb6b10[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.372307:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000009127d001 17ffffc0001001 3:0 ffff8f994f20e700 674 lru 00000080:00200000:2.0:1775764854.372311:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000004bdbd508 674: 1< 0 - - > 2< 2760704 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.372313:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000092eb6b10 00000080:00200000:2.0:1775764854.372314:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 163. 00000080:00200000:2.0:1775764854.372315:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 675 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.372325:0:22506:0:(rw26.c:858:ll_write_end()) pos 2764800, len 4096, copied 4096 00000080:00200000:2.0:1775764854.372328:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000be04333a[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.372331:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000002e96763 17ffffc0001001 3:0 ffff8f994f20e7e0 675 lru 00000080:00200000:2.0:1775764854.372336:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000e80b5658 675: 1< 0 - - > 2< 2764800 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.372338:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000be04333a 00000080:00200000:2.0:1775764854.372339:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 164. 00000080:00200000:2.0:1775764854.372341:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 676 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.372354:0:22506:0:(rw26.c:858:ll_write_end()) pos 2768896, len 4096, copied 4096 00000080:00200000:2.0:1775764854.372358:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000001113891[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.372360:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000002a79030e 17ffffc0001001 3:0 ffff8f994f20e9a0 676 lru 00000080:00200000:2.0:1775764854.372366:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000db16e68a 676: 1< 0 - - > 2< 2768896 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.372367:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000001113891 00000080:00200000:2.0:1775764854.372368:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 165. 00000080:00200000:2.0:1775764854.372370:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 677 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.372385:0:22506:0:(rw26.c:858:ll_write_end()) pos 2772992, len 4096, copied 4096 00000080:00200000:2.0:1775764854.372390:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000029c07a9e[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.372392:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000004a04e69d 17ffffc0001001 3:0 ffff8f994f20eee0 677 lru 00000080:00200000:2.0:1775764854.372398:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000000a09b429 677: 1< 0 - - > 2< 2772992 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.372399:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000029c07a9e 00000080:00200000:2.0:1775764854.372401:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 166. 00000080:00200000:2.0:1775764854.372403:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 678 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.372416:0:22506:0:(rw26.c:858:ll_write_end()) pos 2777088, len 4096, copied 4096 00000080:00200000:2.0:1775764854.372420:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000de2afc41[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.372422:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000dceb2dd8 17ffffc0001001 3:0 ffff8f994f20ea80 678 lru 00000080:00200000:2.0:1775764854.372427:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000e610c6a1 678: 1< 0 - - > 2< 2777088 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.372428:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000de2afc41 00000080:00200000:2.0:1775764854.372429:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 167. 00000080:00200000:2.0:1775764854.372431:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 679 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.372444:0:22506:0:(rw26.c:858:ll_write_end()) pos 2781184, len 4096, copied 4096 00000080:00200000:2.0:1775764854.372448:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000030bc6b75[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.372450:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000aba59008 17ffffc0001001 3:0 ffff8f994f20e540 679 lru 00000080:00200000:2.0:1775764854.372455:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000068d1ab2c 679: 1< 0 - - > 2< 2781184 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.372457:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000030bc6b75 00000080:00200000:2.0:1775764854.372458:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 168. 00000080:00200000:2.0:1775764854.372459:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 680 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.372473:0:22506:0:(rw26.c:858:ll_write_end()) pos 2785280, len 4096, copied 4096 00000080:00200000:2.0:1775764854.372477:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000de8dcb46[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.372479:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000009cc29321 17ffffc0001001 3:0 ffff8f994f20e460 680 lru 00000080:00200000:2.0:1775764854.372483:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000ef72d87e 680: 1< 0 - - > 2< 2785280 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.372485:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000de8dcb46 00000080:00200000:2.0:1775764854.372487:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 169. 00000080:00200000:2.0:1775764854.372488:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 681 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.372500:0:22506:0:(rw26.c:858:ll_write_end()) pos 2789376, len 4096, copied 4096 00000080:00200000:2.0:1775764854.372503:0:22506:0:(rw26.c:916:ll_write_end()) page@000000002ca8af67[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.372506:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000b1710f79 17ffffc0001001 3:0 ffff8f994f20e000 681 lru 00000080:00200000:2.0:1775764854.372510:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000077fd3e0e 681: 1< 0 - - > 2< 2789376 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.372512:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000002ca8af67 00000080:00200000:2.0:1775764854.372513:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 170. 00000080:00200000:2.0:1775764854.372515:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 682 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.372528:0:22506:0:(rw26.c:858:ll_write_end()) pos 2793472, len 4096, copied 4096 00000080:00200000:2.0:1775764854.372532:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000cca7faa6[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.372535:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000f31ade79 17ffffc0001001 3:0 ffff8f994f20e1c0 682 lru 00000080:00200000:2.0:1775764854.372541:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000097a4ef41 682: 1< 0 - - > 2< 2793472 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.372542:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000cca7faa6 00000080:00200000:2.0:1775764854.372543:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 171. 00000080:00200000:2.0:1775764854.372545:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 683 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.372558:0:22506:0:(rw26.c:858:ll_write_end()) pos 2797568, len 4096, copied 4096 00000080:00200000:2.0:1775764854.372561:0:22506:0:(rw26.c:916:ll_write_end()) page@000000003e2d31ad[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.372564:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000b97dc10b 17ffffc0001001 3:0 ffff8f994f20e620 683 lru 00000080:00200000:2.0:1775764854.372569:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000008797f3b9 683: 1< 0 - - > 2< 2797568 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.372571:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000003e2d31ad 00000080:00200000:2.0:1775764854.372572:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 172. 00000080:00200000:2.0:1775764854.372574:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 684 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.372585:0:22506:0:(rw26.c:858:ll_write_end()) pos 2801664, len 4096, copied 4096 00000080:00200000:2.0:1775764854.372589:0:22506:0:(rw26.c:916:ll_write_end()) page@000000002821eecf[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.372591:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000a290f3df 17ffffc0001001 3:0 ffff8f994f20ee00 684 lru 00000080:00200000:2.0:1775764854.372595:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000006cf6970d 684: 1< 0 - - > 2< 2801664 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.372597:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000002821eecf 00000080:00200000:2.0:1775764854.372598:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 173. 00000080:00200000:2.0:1775764854.372600:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 685 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.372711:0:22506:0:(rw26.c:858:ll_write_end()) pos 2805760, len 4096, copied 4096 00000080:00200000:2.0:1775764854.372716:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000006fa111[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.372719:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000815d339a 17ffffc0001001 3:0 ffff8f994f20ec40 685 lru 00000080:00200000:2.0:1775764854.372724:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000002166b4a3 685: 1< 0 - - > 2< 2805760 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.372726:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000006fa111 00000080:00200000:2.0:1775764854.372727:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 174. 00000080:00200000:2.0:1775764854.372729:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 686 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.372832:0:22506:0:(rw26.c:858:ll_write_end()) pos 2809856, len 4096, copied 4096 00000080:00200000:2.0:1775764854.372836:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000b12a7add[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.372838:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000008ca44c65 17ffffc0001001 3:0 ffff8f994f20e8c0 686 lru 00000080:00200000:2.0:1775764854.372842:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000270a05f1 686: 1< 0 - - > 2< 2809856 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.372855:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000b12a7add 00000080:00200000:2.0:1775764854.372857:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 175. 00000080:00200000:2.0:1775764854.372859:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 687 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.372880:0:22506:0:(rw26.c:858:ll_write_end()) pos 2813952, len 4096, copied 4096 00000080:00200000:2.0:1775764854.372884:0:22506:0:(rw26.c:916:ll_write_end()) page@000000002d2dc547[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.372886:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000009cff98d5 17ffffc0001021 2:0 ffff8f994f20ed20 687 lru 00000080:00200000:2.0:1775764854.372891:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000f01e5b50 687: 1< 0 - - > 2< 2813952 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.372892:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000002d2dc547 00000080:00200000:2.0:1775764854.372893:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 176. 00000080:00200000:2.0:1775764854.372895:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 688 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.372909:0:22506:0:(rw26.c:858:ll_write_end()) pos 2818048, len 4096, copied 4096 00000080:00200000:2.0:1775764854.372912:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000558ccef4[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.372914:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000006bcd821c 17ffffc0001001 3:0 ffff8f994f20e0e0 688 lru 00000080:00200000:2.0:1775764854.372919:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000001de21bf4 688: 1< 0 - - > 2< 2818048 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.372920:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000558ccef4 00000080:00200000:2.0:1775764854.372921:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 177. 00000080:00200000:2.0:1775764854.372923:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 689 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.372955:0:22506:0:(rw26.c:858:ll_write_end()) pos 2822144, len 4096, copied 4096 00000080:00200000:2.0:1775764854.372959:0:22506:0:(rw26.c:916:ll_write_end()) page@000000002237acd6[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.372961:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000279bde94 17ffffc0001001 3:0 ffff8f9970b98460 689 lru 00000080:00200000:2.0:1775764854.372966:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000088e2e4db 689: 1< 0 - - > 2< 2822144 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.372968:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000002237acd6 00000080:00200000:2.0:1775764854.372969:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 178. 00000080:00200000:2.0:1775764854.372970:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 690 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.372986:0:22506:0:(rw26.c:858:ll_write_end()) pos 2826240, len 4096, copied 4096 00000080:00200000:2.0:1775764854.372989:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000e80dc484[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.372992:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000233da06e 17ffffc0001001 3:0 ffff8f9970b98000 690 lru 00000080:00200000:2.0:1775764854.372996:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000088ba3896 690: 1< 0 - - > 2< 2826240 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.372998:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000e80dc484 00000080:00200000:2.0:1775764854.372999:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 179. 00000080:00200000:2.0:1775764854.373000:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 691 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.373013:0:22506:0:(rw26.c:858:ll_write_end()) pos 2830336, len 4096, copied 4096 00000080:00200000:2.0:1775764854.373016:0:22506:0:(rw26.c:916:ll_write_end()) page@000000003dea09c2[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.373020:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000c580e07f 17ffffc0001001 3:0 ffff8f9970b981c0 691 lru 00000080:00200000:2.0:1775764854.373025:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000008a68710b 691: 1< 0 - - > 2< 2830336 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.373027:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000003dea09c2 00000080:00200000:2.0:1775764854.373028:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 180. 00000080:00200000:2.0:1775764854.373030:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 692 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.373044:0:22506:0:(rw26.c:858:ll_write_end()) pos 2834432, len 4096, copied 4096 00000080:00200000:2.0:1775764854.373048:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000aecbe6ce[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.373050:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000713dfcf5 17ffffc0001001 3:0 ffff8f9970b98620 692 lru 00000080:00200000:2.0:1775764854.373055:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000004a8bbfb0 692: 1< 0 - - > 2< 2834432 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.373057:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000aecbe6ce 00000080:00200000:2.0:1775764854.373058:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 181. 00000080:00200000:2.0:1775764854.373060:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 693 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.373075:0:22506:0:(rw26.c:858:ll_write_end()) pos 2838528, len 4096, copied 4096 00000080:00200000:2.0:1775764854.373078:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000060071521[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.373081:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000009dda70c1 17ffffc0001001 3:0 ffff8f9970b98e00 693 lru 00000080:00200000:2.0:1775764854.373086:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000d4ec773f 693: 1< 0 - - > 2< 2838528 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.373087:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000060071521 00000080:00200000:2.0:1775764854.373089:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 182. 00000080:00200000:2.0:1775764854.373090:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 694 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.373103:0:22506:0:(rw26.c:858:ll_write_end()) pos 2842624, len 4096, copied 4096 00000080:00200000:2.0:1775764854.373106:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000fbbc809c[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.373109:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000003c71d94e 17ffffc0001001 3:0 ffff8f9970b98c40 694 lru 00000080:00200000:2.0:1775764854.373113:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000007b5c8b8e 694: 1< 0 - - > 2< 2842624 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.373115:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000fbbc809c 00000080:00200000:2.0:1775764854.373116:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 183. 00000080:00200000:2.0:1775764854.373117:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 695 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.373130:0:22506:0:(rw26.c:858:ll_write_end()) pos 2846720, len 4096, copied 4096 00000080:00200000:2.0:1775764854.373133:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000095e05cc5[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.373136:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000e00951f1 17ffffc0001001 3:0 ffff8f9970b988c0 695 lru 00000080:00200000:2.0:1775764854.373141:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000084b77bf 695: 1< 0 - - > 2< 2846720 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.373143:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000095e05cc5 00000080:00200000:2.0:1775764854.373144:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 184. 00000080:00200000:2.0:1775764854.373146:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 696 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.373159:0:22506:0:(rw26.c:858:ll_write_end()) pos 2850816, len 4096, copied 4096 00000080:00200000:2.0:1775764854.373163:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000011a9c3c7[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.373166:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000003c7fa130 17ffffc0001001 3:0 ffff8f9970b98d20 696 lru 00000080:00200000:2.0:1775764854.373251:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000009d699918 696: 1< 0 - - > 2< 2850816 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.373253:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000011a9c3c7 00000080:00200000:2.0:1775764854.373254:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 185. 00000080:00200000:2.0:1775764854.373257:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 697 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.373271:0:22506:0:(rw26.c:858:ll_write_end()) pos 2854912, len 4096, copied 4096 00000080:00200000:2.0:1775764854.373275:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000032224a06[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.373278:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000d4200272 17ffffc0001001 3:0 ffff8f9970b980e0 697 lru 00000080:00200000:2.0:1775764854.373283:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000067d95721 697: 1< 0 - - > 2< 2854912 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.373284:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000032224a06 00000080:00200000:2.0:1775764854.373285:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 186. 00000080:00200000:2.0:1775764854.373287:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 698 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.373299:0:22506:0:(rw26.c:858:ll_write_end()) pos 2859008, len 4096, copied 4096 00000080:00200000:2.0:1775764854.373302:0:22506:0:(rw26.c:916:ll_write_end()) page@000000006557f9dc[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.373305:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000008e6da7cf 17ffffc0001001 3:0 ffff8f9970b982a0 698 lru 00000080:00200000:2.0:1775764854.373310:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000d002c1d4 698: 1< 0 - - > 2< 2859008 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.373312:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000006557f9dc 00000080:00200000:2.0:1775764854.373313:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 187. 00000080:00200000:2.0:1775764854.373314:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 699 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.373324:0:22506:0:(rw26.c:858:ll_write_end()) pos 2863104, len 4096, copied 4096 00000080:00200000:2.0:1775764854.373327:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000074db952c[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.373329:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000a5cbe67a 17ffffc0001001 3:0 ffff8f9970b98380 699 lru 00000080:00200000:2.0:1775764854.373333:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000c71739d0 699: 1< 0 - - > 2< 2863104 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.373335:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000074db952c 00000080:00200000:2.0:1775764854.373336:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 188. 00000080:00200000:2.0:1775764854.373338:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 700 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.373348:0:22506:0:(rw26.c:858:ll_write_end()) pos 2867200, len 4096, copied 4096 00000080:00200000:2.0:1775764854.373351:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000637c9c2d[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.373355:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000a0c7167e 17ffffc0001001 3:0 ffff8f9970b98b60 700 lru 00000080:00200000:2.0:1775764854.373359:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000008b277197 700: 1< 0 - - > 2< 2867200 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.373361:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000637c9c2d 00000080:00200000:2.0:1775764854.373362:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 189. 00000080:00200000:2.0:1775764854.373364:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 701 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.373377:0:22506:0:(rw26.c:858:ll_write_end()) pos 2871296, len 4096, copied 4096 00000080:00200000:2.0:1775764854.373381:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000086513a8c[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.373383:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000c20a6edc 17ffffc0001001 3:0 ffff8f9970b98700 701 lru 00000080:00200000:2.0:1775764854.373389:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000cf2d5769 701: 1< 0 - - > 2< 2871296 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.373390:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000086513a8c 00000080:00200000:2.0:1775764854.373392:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 190. 00000080:00200000:2.0:1775764854.373394:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 702 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.373410:0:22506:0:(rw26.c:858:ll_write_end()) pos 2875392, len 4096, copied 4096 00000080:00200000:2.0:1775764854.373414:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000ffb70297[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.373416:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000dddff576 17ffffc0001021 2:0 ffff8f9970b987e0 702 lru 00000080:00200000:2.0:1775764854.373421:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000039477786 702: 1< 0 - - > 2< 2875392 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.373422:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000ffb70297 00000080:00200000:2.0:1775764854.373423:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 191. 00000080:00200000:2.0:1775764854.373425:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 703 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.373441:0:22506:0:(rw26.c:858:ll_write_end()) pos 2879488, len 4096, copied 4096 00000080:00200000:2.0:1775764854.373445:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000d796d90d[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.373448:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000246d8a95 17ffffc0001001 3:0 ffff8f9970b989a0 703 lru 00000080:00200000:2.0:1775764854.373452:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000842e337a 703: 1< 0 - - > 2< 2879488 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.373455:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000d796d90d 00000080:00200000:2.0:1775764854.373456:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 192. 00000080:00200000:2.0:1775764854.373458:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 704 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.373476:0:22506:0:(rw26.c:858:ll_write_end()) pos 2883584, len 4096, copied 4096 00000080:00200000:2.0:1775764854.373480:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000000ace8b6[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.373482:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000070dda469 17ffffc0001001 3:0 ffff8f9970b98ee0 704 lru 00000080:00200000:2.0:1775764854.373487:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000a021833b 704: 1< 0 - - > 2< 2883584 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.373489:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000000ace8b6 00000080:00200000:2.0:1775764854.373490:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 193. 00000080:00200000:2.0:1775764854.373492:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 705 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.373507:0:22506:0:(rw26.c:858:ll_write_end()) pos 2887680, len 4096, copied 4096 00000080:00200000:2.0:1775764854.373511:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000074a49c3e[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.373513:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000c3c3fccf 17ffffc0001001 3:0 ffff8f9970b98a80 705 lru 00000080:00200000:2.0:1775764854.373518:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000075e596f2 705: 1< 0 - - > 2< 2887680 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.373519:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000074a49c3e 00000080:00200000:2.0:1775764854.373520:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 194. 00000080:00200000:2.0:1775764854.373522:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 706 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.373534:0:22506:0:(rw26.c:858:ll_write_end()) pos 2891776, len 4096, copied 4096 00000080:00200000:2.0:1775764854.373536:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000edeb37e0[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.373539:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000008dbb2c7a 17ffffc0001001 3:0 ffff8f9970b98540 706 lru 00000080:00200000:2.0:1775764854.373543:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000070c083ed 706: 1< 0 - - > 2< 2891776 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.373545:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000edeb37e0 00000080:00200000:2.0:1775764854.373546:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 195. 00000080:00200000:2.0:1775764854.373547:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 707 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.373577:0:22506:0:(rw26.c:858:ll_write_end()) pos 2895872, len 4096, copied 4096 00000080:00200000:2.0:1775764854.373581:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000c8118299[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.373583:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000009308af43 17ffffc0001001 3:0 ffff8f9970f93620 707 lru 00000080:00200000:2.0:1775764854.373588:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000bcc4f152 707: 1< 0 - - > 2< 2895872 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.373590:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000c8118299 00000080:00200000:2.0:1775764854.373591:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 196. 00000080:00200000:2.0:1775764854.373593:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 708 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.373605:0:22506:0:(rw26.c:858:ll_write_end()) pos 2899968, len 4096, copied 4096 00000080:00200000:2.0:1775764854.373608:0:22506:0:(rw26.c:916:ll_write_end()) page@000000007d81640c[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.373702:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000008e9c2626 17ffffc0001001 3:0 ffff8f9970f93e00 708 lru 00000080:00200000:2.0:1775764854.373709:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000b1d14cac 708: 1< 0 - - > 2< 2899968 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.373711:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000007d81640c 00000080:00200000:2.0:1775764854.373712:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 197. 00000080:00200000:2.0:1775764854.373714:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 709 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.373730:0:22506:0:(rw26.c:858:ll_write_end()) pos 2904064, len 4096, copied 4096 00000080:00200000:2.0:1775764854.373734:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000847f5b14[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.373763:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000d96e6615 17ffffc0001001 3:0 ffff8f9970f93c40 709 lru 00000080:00200000:2.0:1775764854.373769:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000bc28613c 709: 1< 0 - - > 2< 2904064 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.373771:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000847f5b14 00000080:00200000:2.0:1775764854.373772:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 198. 00000080:00200000:2.0:1775764854.373774:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 710 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.373790:0:22506:0:(rw26.c:858:ll_write_end()) pos 2908160, len 4096, copied 4096 00000080:00200000:2.0:1775764854.373793:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000511d2c5c[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.373796:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000001905f251 17ffffc0001001 3:0 ffff8f9970f938c0 710 lru 00000080:00200000:2.0:1775764854.373800:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000068464184 710: 1< 0 - - > 2< 2908160 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.373802:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000511d2c5c 00000080:00200000:2.0:1775764854.373803:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 199. 00000080:00200000:2.0:1775764854.373805:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 711 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.373816:0:22506:0:(rw26.c:858:ll_write_end()) pos 2912256, len 4096, copied 4096 00000080:00200000:2.0:1775764854.373819:0:22506:0:(rw26.c:916:ll_write_end()) page@000000006936e76a[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.373821:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000d46d1dfa 17ffffc0001001 3:0 ffff8f9970f93d20 711 lru 00000080:00200000:2.0:1775764854.373826:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000b03212b1 711: 1< 0 - - > 2< 2912256 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.373827:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000006936e76a 00000080:00200000:2.0:1775764854.373828:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 200. 00000080:00200000:2.0:1775764854.373829:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 712 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.373842:0:22506:0:(rw26.c:858:ll_write_end()) pos 2916352, len 4096, copied 4096 00000080:00200000:2.0:1775764854.373858:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000f7a18fa4[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.373860:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000009d39355d 17ffffc0001001 3:0 ffff8f9970f930e0 712 lru 00000080:00200000:2.0:1775764854.373865:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000002259862c 712: 1< 0 - - > 2< 2916352 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.373866:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000f7a18fa4 00000080:00200000:2.0:1775764854.373867:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 201. 00000080:00200000:2.0:1775764854.373869:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 713 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.373883:0:22506:0:(rw26.c:858:ll_write_end()) pos 2920448, len 4096, copied 4096 00000080:00200000:2.0:1775764854.373886:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000a5ba1ba8[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.373888:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000d460971d 17ffffc0001001 3:0 ffff8f9970f932a0 713 lru 00000080:00200000:2.0:1775764854.373893:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000cc2778d4 713: 1< 0 - - > 2< 2920448 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.373894:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000a5ba1ba8 00000080:00200000:2.0:1775764854.373896:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 202. 00000080:00200000:2.0:1775764854.373898:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 714 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.373910:0:22506:0:(rw26.c:858:ll_write_end()) pos 2924544, len 4096, copied 4096 00000080:00200000:2.0:1775764854.373913:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000b707be59[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.373915:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000033434183 17ffffc0001001 3:0 ffff8f9970f93380 714 lru 00000080:00200000:2.0:1775764854.373919:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000f3ebc3fe 714: 1< 0 - - > 2< 2924544 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.373921:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000b707be59 00000080:00200000:2.0:1775764854.373922:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 203. 00000080:00200000:2.0:1775764854.373924:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 715 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.373933:0:22506:0:(rw26.c:858:ll_write_end()) pos 2928640, len 4096, copied 4096 00000080:00200000:2.0:1775764854.373936:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000e7131377[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.373939:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000006eeb8329 17ffffc0001001 3:0 ffff8f9970f93b60 715 lru 00000080:00200000:2.0:1775764854.373944:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000f6395aa9 715: 1< 0 - - > 2< 2928640 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.373945:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000e7131377 00000080:00200000:2.0:1775764854.373946:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 204. 00000080:00200000:2.0:1775764854.373948:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 716 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.373960:0:22506:0:(rw26.c:858:ll_write_end()) pos 2932736, len 4096, copied 4096 00000080:00200000:2.0:1775764854.373963:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000b736cda2[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.373965:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000b73a2a10 17ffffc0001001 3:0 ffff8f9970f93700 716 lru 00000080:00200000:2.0:1775764854.373970:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000001e49f92b 716: 1< 0 - - > 2< 2932736 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.373971:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000b736cda2 00000080:00200000:2.0:1775764854.373972:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 205. 00000080:00200000:2.0:1775764854.373974:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 717 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.373989:0:22506:0:(rw26.c:858:ll_write_end()) pos 2936832, len 4096, copied 4096 00000080:00200000:2.0:1775764854.373992:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000a6456cf4[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.373995:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000e206fb1f 17ffffc0001021 2:0 ffff8f9970f937e0 717 lru 00000080:00200000:2.0:1775764854.373999:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000c52a1379 717: 1< 0 - - > 2< 2936832 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.374001:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000a6456cf4 00000080:00200000:2.0:1775764854.374002:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 206. 00000080:00200000:2.0:1775764854.374003:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 718 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.374014:0:22506:0:(rw26.c:858:ll_write_end()) pos 2940928, len 4096, copied 4096 00000080:00200000:2.0:1775764854.374018:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000f0a9a466[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.374020:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000f46388c5 17ffffc0001001 3:0 ffff8f9970f939a0 718 lru 00000080:00200000:2.0:1775764854.374025:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000f21cb4d2 718: 1< 0 - - > 2< 2940928 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.374027:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000f0a9a466 00000080:00200000:2.0:1775764854.374029:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 207. 00000080:00200000:2.0:1775764854.374030:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 719 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.374045:0:22506:0:(rw26.c:858:ll_write_end()) pos 2945024, len 4096, copied 4096 00000080:00200000:2.0:1775764854.374049:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000ebcde9e2[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.374051:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000006e792e94 17ffffc0001001 3:0 ffff8f9970f93ee0 719 lru 00000080:00200000:2.0:1775764854.374056:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000008270dace 719: 1< 0 - - > 2< 2945024 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.374057:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000ebcde9e2 00000080:00200000:2.0:1775764854.374058:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 208. 00000080:00200000:2.0:1775764854.374060:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 720 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.374072:0:22506:0:(rw26.c:858:ll_write_end()) pos 2949120, len 4096, copied 4096 00000080:00200000:2.0:1775764854.374075:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000041e73b4a[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.374078:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000001a46024d 17ffffc0001001 3:0 ffff8f9970f93a80 720 lru 00000080:00200000:2.0:1775764854.374083:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000024ade96f 720: 1< 0 - - > 2< 2949120 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.374085:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000041e73b4a 00000080:00200000:2.0:1775764854.374086:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 209. 00000080:00200000:2.0:1775764854.374088:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 721 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.374101:0:22506:0:(rw26.c:858:ll_write_end()) pos 2953216, len 4096, copied 4096 00000080:00200000:2.0:1775764854.374105:0:22506:0:(rw26.c:916:ll_write_end()) page@000000008ce54ef7[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.374107:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000003cc494a5 17ffffc0001001 3:0 ffff8f9970f93540 721 lru 00000080:00200000:2.0:1775764854.374112:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000095553d44 721: 1< 0 - - > 2< 2953216 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.374114:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000008ce54ef7 00000080:00200000:2.0:1775764854.374115:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 210. 00000080:00200000:2.0:1775764854.374117:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 722 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.374131:0:22506:0:(rw26.c:858:ll_write_end()) pos 2957312, len 4096, copied 4096 00000080:00200000:2.0:1775764854.374134:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000034b4e811[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.374136:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000ff63a816 17ffffc0001001 3:0 ffff8f9970f93460 722 lru 00000080:00200000:2.0:1775764854.374140:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000064d053dd 722: 1< 0 - - > 2< 2957312 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.374142:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000034b4e811 00000080:00200000:2.0:1775764854.374143:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 211. 00000080:00200000:2.0:1775764854.374145:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 723 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.374157:0:22506:0:(rw26.c:858:ll_write_end()) pos 2961408, len 4096, copied 4096 00000080:00200000:2.0:1775764854.374161:0:22506:0:(rw26.c:916:ll_write_end()) page@000000003c54c602[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.374163:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000f3c2a0cc 17ffffc0001001 3:0 ffff8f9970f93000 723 lru 00000080:00200000:2.0:1775764854.374168:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000027af6ba 723: 1< 0 - - > 2< 2961408 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.374256:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000003c54c602 00000080:00200000:2.0:1775764854.374258:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 212. 00000080:00200000:2.0:1775764854.374260:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 724 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.374278:0:22506:0:(rw26.c:858:ll_write_end()) pos 2965504, len 4096, copied 4096 00000080:00200000:2.0:1775764854.374282:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000a5d1fd15[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.374284:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000003d610ed6 17ffffc0001001 3:0 ffff8f9970f931c0 724 lru 00000080:00200000:2.0:1775764854.374288:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000005a22f122 724: 1< 0 - - > 2< 2965504 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.374290:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000a5d1fd15 00000080:00200000:2.0:1775764854.374291:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 213. 00000080:00200000:2.0:1775764854.374293:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 725 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.374324:0:22506:0:(rw26.c:858:ll_write_end()) pos 2969600, len 4096, copied 4096 00000080:00200000:2.0:1775764854.374328:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000090d653c[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.374331:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000351bf1f9 17ffffc0001001 3:0 ffff8f9970f921c0 725 lru 00000080:00200000:2.0:1775764854.374335:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000001053d806 725: 1< 0 - - > 2< 2969600 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.374337:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000090d653c 00000080:00200000:2.0:1775764854.374338:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 214. 00000080:00200000:2.0:1775764854.374340:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 726 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.374358:0:22506:0:(rw26.c:858:ll_write_end()) pos 2973696, len 4096, copied 4096 00000080:00200000:2.0:1775764854.374363:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000a24c57d5[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.374365:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000008685a007 17ffffc0001001 3:0 ffff8f9970f92620 726 lru 00000080:00200000:2.0:1775764854.374370:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000f32f61e3 726: 1< 0 - - > 2< 2973696 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.374371:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000a24c57d5 00000080:00200000:2.0:1775764854.374373:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 215. 00000080:00200000:2.0:1775764854.374374:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 727 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.374388:0:22506:0:(rw26.c:858:ll_write_end()) pos 2977792, len 4096, copied 4096 00000080:00200000:2.0:1775764854.374392:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000b8415d49[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.374395:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000088a72ce2 17ffffc0001001 3:0 ffff8f9970f92e00 727 lru 00000080:00200000:2.0:1775764854.374400:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000003564201c 727: 1< 0 - - > 2< 2977792 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.374402:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000b8415d49 00000080:00200000:2.0:1775764854.374403:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 216. 00000080:00200000:2.0:1775764854.374404:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 728 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.374417:0:22506:0:(rw26.c:858:ll_write_end()) pos 2981888, len 4096, copied 4096 00000080:00200000:2.0:1775764854.374421:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000da6c6b8b[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.374719:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000006a6a36fb 17ffffc0001001 3:0 ffff8f9970f92c40 728 lru 00000080:00200000:2.0:1775764854.374725:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000007baff961 728: 1< 0 - - > 2< 2981888 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.374727:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000da6c6b8b 00000080:00200000:2.0:1775764854.374728:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 217. 00000080:00200000:2.0:1775764854.374730:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 729 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.375114:0:22506:0:(rw26.c:858:ll_write_end()) pos 2985984, len 4096, copied 4096 00000080:00200000:2.0:1775764854.375120:0:22506:0:(rw26.c:916:ll_write_end()) page@000000001255ed8c[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.375123:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000003da9925f 17ffffc0001001 3:0 ffff8f9970f928c0 729 lru 00000080:00200000:2.0:1775764854.375128:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000b9d6e198 729: 1< 0 - - > 2< 2985984 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.375130:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000001255ed8c 00000080:00200000:2.0:1775764854.375131:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 218. 00000080:00200000:2.0:1775764854.375133:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 730 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.375161:0:22506:0:(rw26.c:858:ll_write_end()) pos 2990080, len 4096, copied 4096 00000080:00200000:2.0:1775764854.375165:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000028dd0acc[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.375175:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000c4e6356c 17ffffc0001001 3:0 ffff8f9970f92d20 730 lru 00000080:00200000:2.0:1775764854.375181:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000857877b8 730: 1< 0 - - > 2< 2990080 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.375182:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000028dd0acc 00000080:00200000:2.0:1775764854.375183:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 219. 00000080:00200000:2.0:1775764854.375186:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 731 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.375199:0:22506:0:(rw26.c:858:ll_write_end()) pos 2994176, len 4096, copied 4096 00000080:00200000:2.0:1775764854.375203:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000a2f0aad4[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.375205:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000f06f71fc 17ffffc0001001 3:0 ffff8f9970f920e0 731 lru 00000080:00200000:2.0:1775764854.375209:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000d2ca0927 731: 1< 0 - - > 2< 2994176 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.375211:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000a2f0aad4 00000080:00200000:2.0:1775764854.375213:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 220. 00000080:00200000:2.0:1775764854.375214:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 732 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.375231:0:22506:0:(rw26.c:858:ll_write_end()) pos 2998272, len 4096, copied 4096 00000080:00200000:2.0:1775764854.375234:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000058fc61b9[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.375236:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000003a20495c 17ffffc0001021 2:0 ffff8f9970f922a0 732 lru 00000080:00200000:2.0:1775764854.375241:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000d8fbe79c 732: 1< 0 - - > 2< 2998272 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.375243:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000058fc61b9 00000080:00200000:2.0:1775764854.375244:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 221. 00000080:00200000:2.0:1775764854.375246:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 733 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.375258:0:22506:0:(rw26.c:858:ll_write_end()) pos 3002368, len 4096, copied 4096 00000080:00200000:2.0:1775764854.375262:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000d6bd7faa[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.375264:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000053f19744 17ffffc0001001 3:0 ffff8f9970f92380 733 lru 00000080:00200000:2.0:1775764854.375269:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000d5fe98ff 733: 1< 0 - - > 2< 3002368 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.375270:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000d6bd7faa 00000080:00200000:2.0:1775764854.375271:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 222. 00000080:00200000:2.0:1775764854.375273:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 734 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.375283:0:22506:0:(rw26.c:858:ll_write_end()) pos 3006464, len 4096, copied 4096 00000080:00200000:2.0:1775764854.375286:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000019e104d8[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.375289:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000c301f912 17ffffc0001001 3:0 ffff8f9970f92b60 734 lru 00000080:00200000:2.0:1775764854.375293:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000003d15e6a6 734: 1< 0 - - > 2< 3006464 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.375295:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000019e104d8 00000080:00200000:2.0:1775764854.375296:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 223. 00000080:00200000:2.0:1775764854.375298:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 735 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.375309:0:22506:0:(rw26.c:858:ll_write_end()) pos 3010560, len 4096, copied 4096 00000080:00200000:2.0:1775764854.375312:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000d6538f46[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.375315:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000011d9d2cc 17ffffc0001001 3:0 ffff8f9970f92700 735 lru 00000080:00200000:2.0:1775764854.375320:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000045b8b7a6 735: 1< 0 - - > 2< 3010560 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.375322:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000d6538f46 00000080:00200000:2.0:1775764854.375323:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 224. 00000080:00200000:2.0:1775764854.375325:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 736 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.375337:0:22506:0:(rw26.c:858:ll_write_end()) pos 3014656, len 4096, copied 4096 00000080:00200000:2.0:1775764854.375340:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000fadc3edc[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.375343:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000006dee3914 17ffffc0001001 3:0 ffff8f9970f927e0 736 lru 00000080:00200000:2.0:1775764854.375347:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000044080cd5 736: 1< 0 - - > 2< 3014656 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.375349:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000fadc3edc 00000080:00200000:2.0:1775764854.375350:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 225. 00000080:00200000:2.0:1775764854.375351:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 737 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.375362:0:22506:0:(rw26.c:858:ll_write_end()) pos 3018752, len 4096, copied 4096 00000080:00200000:2.0:1775764854.375364:0:22506:0:(rw26.c:916:ll_write_end()) page@000000000ae2296c[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.375366:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000001c1c11ff 17ffffc0001001 3:0 ffff8f9970f929a0 737 lru 00000080:00200000:2.0:1775764854.375371:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000b0bbcf0a 737: 1< 0 - - > 2< 3018752 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.375372:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000000ae2296c 00000080:00200000:2.0:1775764854.375373:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 226. 00000080:00200000:2.0:1775764854.375375:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 738 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.375388:0:22506:0:(rw26.c:858:ll_write_end()) pos 3022848, len 4096, copied 4096 00000080:00200000:2.0:1775764854.375392:0:22506:0:(rw26.c:916:ll_write_end()) page@000000000c2a807e[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.375395:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000d91e8537 17ffffc0001001 3:0 ffff8f9970f92ee0 738 lru 00000080:00200000:2.0:1775764854.375400:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000009c6fb424 738: 1< 0 - - > 2< 3022848 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.375402:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000000c2a807e 00000080:00200000:2.0:1775764854.375403:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 227. 00000080:00200000:2.0:1775764854.375404:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 739 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.375418:0:22506:0:(rw26.c:858:ll_write_end()) pos 3026944, len 4096, copied 4096 00000080:00200000:2.0:1775764854.375421:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000c1050614[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.375423:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000d1b3cbad 17ffffc0001001 3:0 ffff8f9970f92a80 739 lru 00000080:00200000:2.0:1775764854.375428:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000d5a4ea67 739: 1< 0 - - > 2< 3026944 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.375431:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000c1050614 00000080:00200000:2.0:1775764854.375431:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 228. 00000080:00200000:2.0:1775764854.375433:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 740 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.375447:0:22506:0:(rw26.c:858:ll_write_end()) pos 3031040, len 4096, copied 4096 00000080:00200000:2.0:1775764854.375451:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000ea01f028[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.375453:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000006bf0b9ff 17ffffc0001001 3:0 ffff8f9970f92540 740 lru 00000080:00200000:2.0:1775764854.375458:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000ef5b780e 740: 1< 0 - - > 2< 3031040 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.375460:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000ea01f028 00000080:00200000:2.0:1775764854.375461:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 229. 00000080:00200000:2.0:1775764854.375463:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 741 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.375476:0:22506:0:(rw26.c:858:ll_write_end()) pos 3035136, len 4096, copied 4096 00000080:00200000:2.0:1775764854.375479:0:22506:0:(rw26.c:916:ll_write_end()) page@000000006dcb6fe5[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.375482:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000860ed04b 17ffffc0001001 3:0 ffff8f9970f92460 741 lru 00000080:00200000:2.0:1775764854.375486:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000000036086d 741: 1< 0 - - > 2< 3035136 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.375488:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000006dcb6fe5 00000080:00200000:2.0:1775764854.375489:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 230. 00000080:00200000:2.0:1775764854.375490:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 742 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.375503:0:22506:0:(rw26.c:858:ll_write_end()) pos 3039232, len 4096, copied 4096 00000080:00200000:2.0:1775764854.375506:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000fb3f508b[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.375508:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000e8812b4d 17ffffc0001001 3:0 ffff8f9970f92000 742 lru 00000080:00200000:2.0:1775764854.375512:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000b24eeba4 742: 1< 0 - - > 2< 3039232 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.375514:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000fb3f508b 00000080:00200000:2.0:1775764854.375515:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 231. 00000080:00200000:2.0:1775764854.375517:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 743 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.375552:0:22506:0:(rw26.c:858:ll_write_end()) pos 3043328, len 4096, copied 4096 00000080:00200000:2.0:1775764854.375555:0:22506:0:(rw26.c:916:ll_write_end()) page@000000006a8ccbeb[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.375558:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000dbd403cb 17ffffc0001001 3:0 ffff8f994f159a80 743 lru 00000080:00200000:2.0:1775764854.375562:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000006ad88068 743: 1< 0 - - > 2< 3043328 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.375564:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000006a8ccbeb 00000080:00200000:2.0:1775764854.375565:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 232. 00000080:00200000:2.0:1775764854.375567:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 744 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.375580:0:22506:0:(rw26.c:858:ll_write_end()) pos 3047424, len 4096, copied 4096 00000080:00200000:2.0:1775764854.375584:0:22506:0:(rw26.c:916:ll_write_end()) page@000000000f707b58[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.375587:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000958f35ea 17ffffc0001001 3:0 ffff8f994f159540 744 lru 00000080:00200000:2.0:1775764854.375592:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000d75eb783 744: 1< 0 - - > 2< 3047424 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.375594:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000000f707b58 00000080:00200000:2.0:1775764854.375595:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 233. 00000080:00200000:2.0:1775764854.375596:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 745 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.375609:0:22506:0:(rw26.c:858:ll_write_end()) pos 3051520, len 4096, copied 4096 00000080:00200000:2.0:1775764854.375911:0:22506:0:(rw26.c:916:ll_write_end()) page@000000002ab87274[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.375930:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000bfddc488 17ffffc0001001 3:0 ffff8f994f159460 745 lru 00000080:00200000:2.0:1775764854.375936:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000008b1650a4 745: 1< 0 - - > 2< 3051520 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.375937:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000002ab87274 00000080:00200000:2.0:1775764854.375938:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 234. 00000080:00200000:2.0:1775764854.375941:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 746 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.375958:0:22506:0:(rw26.c:858:ll_write_end()) pos 3055616, len 4096, copied 4096 00000080:00200000:2.0:1775764854.375962:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000db4d1ea5[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.375964:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000009fe66155 17ffffc0001001 3:0 ffff8f994f159000 746 lru 00000080:00200000:2.0:1775764854.375969:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000575a8b01 746: 1< 0 - - > 2< 3055616 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.375970:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000db4d1ea5 00000080:00200000:2.0:1775764854.375971:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 235. 00000080:00200000:2.0:1775764854.375973:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 747 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.375992:0:22506:0:(rw26.c:858:ll_write_end()) pos 3059712, len 4096, copied 4096 00000080:00200000:2.0:1775764854.375996:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000003692737[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.375998:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000d0f1c357 17ffffc0001021 2:0 ffff8f994f1591c0 747 lru 00000080:00200000:2.0:1775764854.376002:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000b6b3b919 747: 1< 0 - - > 2< 3059712 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.376004:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000003692737 00000080:00200000:2.0:1775764854.376005:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 236. 00000080:00200000:2.0:1775764854.376007:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 748 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.376020:0:22506:0:(rw26.c:858:ll_write_end()) pos 3063808, len 4096, copied 4096 00000080:00200000:2.0:1775764854.376024:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000d7544522[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.376026:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000002c81c3c7 17ffffc0001001 3:0 ffff8f994f159620 748 lru 00000080:00200000:2.0:1775764854.376030:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000008713ff44 748: 1< 0 - - > 2< 3063808 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.376031:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000d7544522 00000080:00200000:2.0:1775764854.376032:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 237. 00000080:00200000:2.0:1775764854.376034:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 749 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.376045:0:22506:0:(rw26.c:858:ll_write_end()) pos 3067904, len 4096, copied 4096 00000080:00200000:2.0:1775764854.376048:0:22506:0:(rw26.c:916:ll_write_end()) page@000000000d7255e2[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.376051:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000864598d5 17ffffc0001001 3:0 ffff8f994f159e00 749 lru 00000080:00200000:2.0:1775764854.376056:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000f548a819 749: 1< 0 - - > 2< 3067904 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.376058:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000000d7255e2 00000080:00200000:2.0:1775764854.376073:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 238. 00000080:00200000:2.0:1775764854.376075:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 750 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.376090:0:22506:0:(rw26.c:858:ll_write_end()) pos 3072000, len 4096, copied 4096 00000080:00200000:2.0:1775764854.376094:0:22506:0:(rw26.c:916:ll_write_end()) page@000000007c82786b[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.376096:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000004ebf179a 17ffffc0001001 3:0 ffff8f994f159c40 750 lru 00000080:00200000:2.0:1775764854.376101:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000009e9eaadc 750: 1< 0 - - > 2< 3072000 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.376103:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000007c82786b 00000080:00200000:2.0:1775764854.376104:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 239. 00000080:00200000:2.0:1775764854.376106:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 751 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.376119:0:22506:0:(rw26.c:858:ll_write_end()) pos 3076096, len 4096, copied 4096 00000080:00200000:2.0:1775764854.376123:0:22506:0:(rw26.c:916:ll_write_end()) page@000000005ad10e42[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.376125:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000157c9d94 17ffffc0001001 3:0 ffff8f994f1598c0 751 lru 00000080:00200000:2.0:1775764854.376129:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000003c9513d1 751: 1< 0 - - > 2< 3076096 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.376131:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000005ad10e42 00000080:00200000:2.0:1775764854.376133:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 240. 00000080:00200000:2.0:1775764854.376134:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 752 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.376146:0:22506:0:(rw26.c:858:ll_write_end()) pos 3080192, len 4096, copied 4096 00000080:00200000:2.0:1775764854.376150:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000081b47acc[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.376153:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000b940b05b 17ffffc0001001 3:0 ffff8f994f159d20 752 lru 00000080:00200000:2.0:1775764854.376158:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000012023642 752: 1< 0 - - > 2< 3080192 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.376160:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000081b47acc 00000080:00200000:2.0:1775764854.376161:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 241. 00000080:00200000:2.0:1775764854.376163:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 753 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.376188:0:22506:0:(rw26.c:858:ll_write_end()) pos 3084288, len 4096, copied 4096 00000080:00200000:2.0:1775764854.376193:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000816b6688[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.376195:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000077ee3ecf 17ffffc0001001 3:0 ffff8f994f1590e0 753 lru 00000080:00200000:2.0:1775764854.376200:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000000bfdd89a 753: 1< 0 - - > 2< 3084288 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.376203:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000816b6688 00000080:00200000:2.0:1775764854.376204:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 242. 00000080:00200000:2.0:1775764854.376205:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 754 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.376221:0:22506:0:(rw26.c:858:ll_write_end()) pos 3088384, len 4096, copied 4096 00000080:00200000:2.0:1775764854.376224:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000be784849[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.376240:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000008bc270a5 17ffffc0001001 3:0 ffff8f994f1592a0 754 lru 00000080:00200000:2.0:1775764854.376245:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000490611f5 754: 1< 0 - - > 2< 3088384 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.376247:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000be784849 00000080:00200000:2.0:1775764854.376248:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 243. 00000080:00200000:2.0:1775764854.376251:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 755 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.376265:0:22506:0:(rw26.c:858:ll_write_end()) pos 3092480, len 4096, copied 4096 00000080:00200000:2.0:1775764854.376268:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000a21bd8c0[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.376271:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000f9ea2300 17ffffc0001001 3:0 ffff8f994f159380 755 lru 00000080:00200000:2.0:1775764854.376275:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000cbbf3763 755: 1< 0 - - > 2< 3092480 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.376278:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000a21bd8c0 00000080:00200000:2.0:1775764854.376279:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 244. 00000080:00200000:2.0:1775764854.376280:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 756 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.376293:0:22506:0:(rw26.c:858:ll_write_end()) pos 3096576, len 4096, copied 4096 00000080:00200000:2.0:1775764854.376297:0:22506:0:(rw26.c:916:ll_write_end()) page@000000008577c89e[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.376300:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000035ccf300 17ffffc0001001 3:0 ffff8f994f159b60 756 lru 00000080:00200000:2.0:1775764854.376305:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000f4e88592 756: 1< 0 - - > 2< 3096576 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.376307:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000008577c89e 00000080:00200000:2.0:1775764854.376308:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 245. 00000080:00200000:2.0:1775764854.376310:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 757 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.376322:0:22506:0:(rw26.c:858:ll_write_end()) pos 3100672, len 4096, copied 4096 00000080:00200000:2.0:1775764854.376326:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000795d9a39[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.376329:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000101d4b94 17ffffc0001001 3:0 ffff8f994f159700 757 lru 00000080:00200000:2.0:1775764854.376335:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000008ec3c80f 757: 1< 0 - - > 2< 3100672 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.376337:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000795d9a39 00000080:00200000:2.0:1775764854.376339:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 246. 00000080:00200000:2.0:1775764854.376340:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 758 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.376351:0:22506:0:(rw26.c:858:ll_write_end()) pos 3104768, len 4096, copied 4096 00000080:00200000:2.0:1775764854.376355:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000775525f2[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.376357:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000000aa3bdd6 17ffffc0001001 3:0 ffff8f994f1597e0 758 lru 00000080:00200000:2.0:1775764854.376362:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000891756ba 758: 1< 0 - - > 2< 3104768 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.376364:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000775525f2 00000080:00200000:2.0:1775764854.376377:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 247. 00000080:00200000:2.0:1775764854.376379:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 759 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.376395:0:22506:0:(rw26.c:858:ll_write_end()) pos 3108864, len 4096, copied 4096 00000080:00200000:2.0:1775764854.376399:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000024183582[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.376402:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000009a6e7a86 17ffffc0001001 3:0 ffff8f994f1599a0 759 lru 00000080:00200000:2.0:1775764854.376406:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000009430283 759: 1< 0 - - > 2< 3108864 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.376408:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000024183582 00000080:00200000:2.0:1775764854.376409:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 248. 00000080:00200000:2.0:1775764854.376411:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 760 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.376423:0:22506:0:(rw26.c:858:ll_write_end()) pos 3112960, len 4096, copied 4096 00000080:00200000:2.0:1775764854.376427:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000e7b33754[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.376429:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000034a06e10 17ffffc0001001 3:0 ffff8f994f159ee0 760 lru 00000080:00200000:2.0:1775764854.376435:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000c7ea3c84 760: 1< 0 - - > 2< 3112960 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.376437:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000e7b33754 00000080:00200000:2.0:1775764854.376438:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 249. 00000080:00200000:2.0:1775764854.376440:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 761 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.376468:0:22506:0:(rw26.c:858:ll_write_end()) pos 3117056, len 4096, copied 4096 00000080:00200000:2.0:1775764854.376472:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000d54094f8[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.376474:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @0000000067544982 17ffffc0001001 3:0 ffff8f9970f90b60 761 lru 00000080:00200000:2.0:1775764854.376479:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000f9f55f4f 761: 1< 0 - - > 2< 3117056 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.376481:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000d54094f8 00000080:00200000:2.0:1775764854.376482:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 250. 00000080:00200000:2.0:1775764854.376484:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 762 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.376504:0:22506:0:(rw26.c:858:ll_write_end()) pos 3121152, len 4096, copied 4096 00000080:00200000:2.0:1775764854.376508:0:22506:0:(rw26.c:916:ll_write_end()) page@000000002418ea15[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.376511:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000e2f3e7b4 17ffffc0001021 2:0 ffff8f9970f90700 762 lru 00000080:00200000:2.0:1775764854.376516:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000e40a5b99 762: 1< 0 - - > 2< 3121152 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.376518:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000002418ea15 00000080:00200000:2.0:1775764854.376519:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 251. 00000080:00200000:2.0:1775764854.376521:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 763 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.376536:0:22506:0:(rw26.c:858:ll_write_end()) pos 3125248, len 4096, copied 4096 00000080:00200000:2.0:1775764854.376540:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000fbb642c8[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.376555:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000001d306566 17ffffc0001001 3:0 ffff8f9970f907e0 763 lru 00000080:00200000:2.0:1775764854.376560:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@000000001754a784 763: 1< 0 - - > 2< 3125248 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.376561:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000fbb642c8 00000080:00200000:2.0:1775764854.376562:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 252. 00000080:00200000:2.0:1775764854.376564:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 764 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.376579:0:22506:0:(rw26.c:858:ll_write_end()) pos 3129344, len 4096, copied 4096 00000080:00200000:2.0:1775764854.376583:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000098cf2f16[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.376586:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000004d661de1 17ffffc0001001 3:0 ffff8f9970f909a0 764 lru 00000080:00200000:2.0:1775764854.376591:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000f7e53108 764: 1< 0 - - > 2< 3129344 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.376593:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000098cf2f16 00000080:00200000:2.0:1775764854.376594:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 253. 00000080:00200000:2.0:1775764854.376596:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 765 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.377332:0:22506:0:(rw26.c:858:ll_write_end()) pos 3133440, len 4096, copied 4096 00000080:00200000:2.0:1775764854.377341:0:22506:0:(rw26.c:916:ll_write_end()) page@000000003102a83b[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.377344:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000d88ea74f 17ffffc0001001 3:0 ffff8f9970f90ee0 765 lru 00000080:00200000:2.0:1775764854.377350:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000970ce298 765: 1< 0 - - > 2< 3133440 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.377351:0:22506:0:(rw26.c:916:ll_write_end()) end page@000000003102a83b 00000080:00200000:2.0:1775764854.377352:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 254. 00000080:00200000:2.0:1775764854.377355:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 766 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.377382:0:22506:0:(rw26.c:858:ll_write_end()) pos 3137536, len 4096, copied 4096 00000080:00200000:2.0:1775764854.377386:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000bba9a2b9[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.377388:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000002bbb4504 17ffffc0001001 3:0 ffff8f9970f90a80 766 lru 00000080:00200000:2.0:1775764854.377393:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@0000000090b390d9 766: 1< 0 - - > 2< 3137536 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.377394:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000bba9a2b9 00000080:00200000:2.0:1775764854.377395:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 255. 00000080:00200000:2.0:1775764854.377397:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 767 of 0 to 4096 bytes 00000080:00200000:2.0:1775764854.377411:0:22506:0:(rw26.c:858:ll_write_end()) pos 3141632, len 4096, copied 4096 00000080:00200000:2.0:1775764854.377415:0:22506:0:(rw26.c:916:ll_write_end()) page@00000000b52bb6bd[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.377417:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @000000006d49e8d3 17ffffc0001001 3:0 ffff8f9970f90540 767 lru 00000080:00200000:2.0:1775764854.377422:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000cc6b6e7d 767: 1< 0 - - > 2< 3141632 0 0 0x0 0x0 | 000000004b957d6e 000000008927ecb9 > 3< 0 0 > 4< 0 0 8 466354176 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.377424:0:22506:0:(rw26.c:916:ll_write_end()) end page@00000000b52bb6bd 00000080:00200000:2.0:1775764854.377444:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 256. 00000080:00200000:2.0:1775764854.377446:0:22506:0:(rw26.c:663:ll_write_begin()) Writing 768 of 0 to 1 bytes 00000080:00200000:2.0:1775764854.377477:0:22506:0:(rw26.c:858:ll_write_end()) pos 3145728, len 1, copied 1 00000080:00200000:2.0:1775764854.377481:0:22506:0:(rw26.c:916:ll_write_end()) page@0000000086b0bd2b[3 00000000a87f46e8 2 1 000000008ee81b18] 00000080:00200000:2.0:1775764854.377484:0:22506:0:(rw26.c:916:ll_write_end()) vmpage @00000000ba350b12 17ffffc0001001 3:0 ffff8f9970f90460 768 lru 00000080:00200000:2.0:1775764854.377488:0:22506:0:(rw26.c:916:ll_write_end()) osc-page@00000000ed1db186 768: 1< 0 - - > 2< 3145728 0 0 0x0 0x0 | 00000000f240ca6f 00000000f7eef48d > 3< 0 0 > 4< 0 0 8 412352512 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764854.377490:0:22506:0:(rw26.c:916:ll_write_end()) end page@0000000086b0bd2b 00000080:00200000:2.0:1775764854.377491:0:22506:0:(rw26.c:916:ll_write_end()) queued cl_page: 257. 00000080:00200000:2.0:1775764854.377495:0:22506:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 257, from 1024, to 1 prio 0 00000008:00010000:2.0:1775764854.377559:0:22506:0:(osc_cache.c:848:osc_extent_find()) ### extent: 00000000bb37d7a5 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000080:00200000:2.0:1775764854.377593:0:22506:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8f9952eb75f8, count 15, dirtied 15 00000080:00200000:2.0:1775764854.377608:0:22506:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8f9952eb75f8, count 15, dirtied 15 00000080:00200000:2.0:1775764854.378160:0:22506:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8f9952eb75f8, count 15, dirtied 15 00000080:00200000:2.0:1775764854.378473:0:22506:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8f9952eb75f8, count 15, dirtied 15 00000080:00200000:2.0:1775764854.378492:0:22506:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8f9952eb75f8, count 15, dirtied 15 00000080:00200000:2.0:1775764854.378505:0:22506:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8f9952eb75f8, count 15, dirtied 15 00000080:00200000:2.0:1775764854.378518:0:22506:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8f9952eb75f8, count 15, dirtied 15 00000080:00200000:2.0:1775764854.378534:0:22506:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8f9952eb75f8, count 15, dirtied 15 00000080:00200000:2.0:1775764854.378551:0:22506:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8f9952eb75f8, count 15, dirtied 15 00000080:00200000:2.0:1775764854.378568:0:22506:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8f9952eb75f8, count 15, dirtied 15 00000080:00200000:2.0:1775764854.378583:0:22506:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8f9952eb75f8, count 15, dirtied 15 00000080:00200000:2.0:1775764854.378600:0:22506:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8f9952eb75f8, count 15, dirtied 15 00000080:00200000:2.0:1775764854.378836:0:22506:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8f9952eb75f8, count 15, dirtied 15 00000080:00200000:2.0:1775764854.378864:0:22506:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8f9952eb75f8, count 15, dirtied 15 00000080:00200000:2.0:1775764854.378882:0:22506:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8f9952eb75f8, count 15, dirtied 15 00000080:00200000:2.0:1775764854.378900:0:22506:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8f9952eb75f8, count 15, dirtied 15 00000080:00200000:2.0:1775764854.378917:0:22506:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8f9952eb75f8, count 15, dirtied 15 00000080:00200000:2.0:1775764854.378924:0:22506:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8f9952eb75f8, count 1, dirtied 1 00000008:00010000:2.0:1775764854.379007:0:20870:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### extent: 00000000bb37d7a5 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000008:00010000:2.0:1775764854.379050:0:20870:0:(osc_cache.c:1167:osc_extent_make_ready()) ### extent: 00000000bb37d7a5 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764854.379582:0:20870:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (2097152 2101247) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000008:00100000:2.0:1775764854.380720:0:20870:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8f99703d3b80 - object 0x280000403:1796 offset 2097152<>3145728 00000008:00010000:3.0:1775764854.380941:0:22506:0:(osc_cache.c:848:osc_extent_find()) ### extent: 000000003deacf37 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000587dd0e8/0x36f9bdc9e421ca0f lrc: 4/0,1 mode: PW/PW res: [0x240000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc4ab expref: -99 pid: 22506 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:3.0:1775764854.380967:0:22506:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8f9952eb75f8, count 1, dirtied 1 00000080:00200000:3.0:1775764854.380976:0:22506:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 257 pages 1047553 bytes, tot: 1047553 00000080:00200000:3.0:1775764854.380980:0:22506:0:(vvp_io.c:1353:vvp_io_write_start()) f11a.sanity-pfl: write bytes 0, result: 1047553 00000020:00010000:3.0:1775764854.381002:0:22506:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f99705ba998 (ffff8f99705b8878/0) at cl_lock_release():245 00000020:00010000:3.0:1775764854.381006:0:22506:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99705ba998 (ffff8f99705b8878/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764854.381010:0:22506:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99708df030 (ffff8f99703147e8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764854.381017:0:22506:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 4/0,1 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1775764854.381028:0:22506:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 3/0,0 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000020:00010000:3.0:1775764854.381043:0:22506:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99708df070 (ffff8f99703141b8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764854.381048:0:22506:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 4/0,1 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1775764854.381055:0:22506:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000020:00010000:3.0:1775764854.381063:0:22506:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99708df0b0 (ffff8f9970314128/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764854.381068:0:22506:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:3.0:1775764854.381081:0:22506:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 4/0,0 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00000020:00010000:3.0:1775764854.381088:0:22506:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f99708df0f0 (ffff8f99703146c8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764854.381093:0:22506:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000587dd0e8/0x36f9bdc9e421ca0f lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc4ab expref: -99 pid: 22506 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764854.381099:0:22506:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000587dd0e8/0x36f9bdc9e421ca0f lrc: 4/0,0 mode: PW/PW res: [0x240000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc4ab expref: -99 pid: 22506 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:3.0:1775764854.381107:0:22506:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99705ba998 (ffff8f99705b8878/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764854.381110:0:22506:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99708df030 (ffff8f99705b8878/1) at cl_lock_fini():72 00000020:00010000:3.0:1775764854.381114:0:22506:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99708df070 (ffff8f99705b8878/1) at cl_lock_fini():72 00000020:00010000:3.0:1775764854.381117:0:22506:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99708df0b0 (ffff8f99705b8878/1) at cl_lock_fini():72 00000020:00010000:3.0:1775764854.381120:0:22506:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f99708df0f0 (ffff8f99705b8878/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764854.381127:0:22506:0:(vvp_io.c:517:vvp_io_advance()) advancing 1047553 bytes 00000080:00200000:3.0:1775764854.381136:0:22506:0:(file.c:2173:ll_file_io_generic()) Range unlock [0, 18446744073709551615] 00000080:00200000:3.0:1775764854.381202:0:22506:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xee0:0x0] ignore/verify layout 0/0, layout version 8 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764854.381206:0:22506:0:(file.c:2206:ll_file_io_generic()) f11a.sanity-pfl: 2 io complete with rc: 0, result: 1047553, restart: 0 00000080:00200000:3.0:1775764854.381220:0:22506:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0xee0:0x0] updating i_size 3145729 i_blocks 4 00000080:00200000:3.0:1775764854.381224:0:22506:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 1047553 00000080:00200008:3.0:1775764854.382428:0:22506:0:(file.c:447:ll_file_release()) START file f11a.sanity-pfl:[0x200000401:0xee0:0x0](ffff8f9952eb7448), flags 102001 00010000:00010000:3.0:1775764854.382460:0:22506:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421ca01 lrc: 2/0,0 mode: CW/CW res: [0x200000401:0xee0:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc48f expref: -99 pid: 22506 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000100:00100000:2.0:1775764854.388474:0:2381:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1775764854.388501:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967056000:192.168.202.146@tcp:4:rwv.0 00000080:00200000:0.0:1775764854.393069:0:22507:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:0.0:1775764854.393095:0:22507:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764854.393114:0:22507:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 32 lru_type: 1 00010000:00010000:0.0:1775764854.393130:0:22507:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 32 lru_type: 1 00000080:00200000:0.0:1775764854.393137:0:22507:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764854.393145:0:22507:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:0.0:1775764854.393149:0:22507:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:0.0:1775764854.393155:0:22507:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac19c7b58 00010000:00010000:0.0:1775764854.393191:0:22507:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 32 lru_type: 1 00010000:00010000:0.0:1775764854.393200:0:22507:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 33 lru_type: 1 00000080:00200000:0.0:1775764854.393293:0:22507:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:0.0:1775764854.393302:0:22507:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d11a.sanity-pfl, flags=24 00000080:00200000:0.0:1775764854.393308:0:22507:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764854.393319:0:22507:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=f11a.sanity-pfl, dir=[0x200000401:0xedf:0x0](ffff8f9952eedb88), flags=8 00000080:00200000:0.0:1775764854.393323:0:22507:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=f11a.sanity-pfl, dir=[0x200000401:0xedf:0x0](ffff8f9952eedb88), intent=getattr 00000002:00010000:0.0:1775764854.393336:0:22507:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f11a.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000401:0xedf:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764854.393677:0:22507:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421ca16 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 22507 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.393686:0:22507:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421ca16 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 22507 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.393696:0:22507:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421ca16 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 22507 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764854.393722:0:22507:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77b80 pname:cluuid:pid:xid:nid:opc:job ls:3f83546c-01a9-4156-8a97-44c446245d3e:22507:1862022967056128:192.168.202.146@tcp:101:ls.0 00000100:00100000:0.0:1775764854.393818:0:22507:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102480 going to sleep for 11 seconds 00000100:00100000:0.0:1775764854.397619:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764854.397655:0:11131:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978417408 00000100:00100000:0.0:1775764854.397665:0:11131:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978417408 00000100:00100000:0.0:1775764854.397715:0:11131:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24228 00000100:00100000:0.0:1775764854.397724:0:11131:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970a75c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:3f83546c-01a9-4156-8a97-44c446245d3e+5:17505:x1862022978417408:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764854.397758:0:11131:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421ca01 lrc: 2/0,0 mode: CW/CW res: [0x200000401:0xee0:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x2aac6a50e5fc48f expref: -99 pid: 22506 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764854.397877:0:11131:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764854.397893:0:11131:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970a75c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:3f83546c-01a9-4156-8a97-44c446245d3e+5:17505:x1862022978417408:12345-192.168.202.146@tcp:104: Request processed in 162us (274us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764854.397901:0:11131:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24228 00010000:00010000:3.0:1775764854.398342:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1775764854.398352:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421ca01 lrc: 2/0,0 mode: CW/CW res: [0x200000401:0xee0:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x2aac6a50e5fc48f expref: -99 pid: 22506 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764854.398362:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f9970641e00 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:3.0:1775764854.398367:0:19801:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421ca01 lrc: 2/0,0 mode: CW/CW res: [0x200000401:0xee0:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x2aac6a50e5fc48f expref: -99 pid: 22506 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764854.398376:0:19801:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421ca01 lrc: 2/0,0 mode: CW/CW res: [0x200000401:0xee0:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x2aac6a50e5fc48f expref: -99 pid: 22506 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1775764854.398386:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421ca01 lrc: 3/0,0 mode: CW/CW res: [0x200000401:0xee0:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x408400000000 nid: local remote: 0x2aac6a50e5fc48f expref: -99 pid: 22506 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000002:00100000:3.0:1775764854.398452:0:19801:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f994f2e6d80 x1862022967055104/t8589934632(8589934632) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1775764870 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'rwv.0' uid:0 gid:0 projid:0 00000020:00100000:3.0:1775764854.398465:0:19801:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:3.0:1775764854.398470:0:19801:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970edf100 x1862022967056384/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/1048 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:3.0:1775764854.398495:0:19801:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ldlm_bl_06:3f83546c-01a9-4156-8a97-44c446245d3e:19801:1862022967056384:192.168.202.146@tcp:35:ldlm_bl.0 00000100:00100000:3.0:1775764854.398683:0:19801:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994ecbb000 going to sleep for 11 seconds 00000100:00100000:3.0:1775764854.407255:0:19801:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ldlm_bl_06:3f83546c-01a9-4156-8a97-44c446245d3e:19801:1862022967056384:192.168.202.146@tcp:35:ldlm_bl.0 00000002:00100000:0.0:1775764854.407337:0:19801:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f994f2e6d80 x1862022967055104/t8589934632(8589934632) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1775764870 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'rwv.0' uid:0 gid:0 projid:0 00010000:00010000:0.0:1775764854.408074:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421ca01 lrc: 2/0,0 mode: --/CW res: [0x200000401:0xee0:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x2aac6a50e5fc48f expref: -99 pid: 22506 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764854.408086:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:1.0:1775764854.408167:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967056512:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1775764854.408287:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421ca01 lrc: 1/0,0 mode: --/CW res: [0x200000401:0xee0:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x2aac6a50e5fc48f expref: -99 pid: 22506 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764854.408297:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970641e00/0x36f9bdc9e421ca01 lrc: 0/0,0 mode: --/CW res: [0x200000401:0xee0:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x2aac6a50e5fc48f expref: -99 pid: 22506 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1775764854.408315:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764854.408317:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764854.408319:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764854.411341:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967056512:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1775764854.411548:0:22507:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77b80 pname:cluuid:pid:xid:nid:opc:job ls:3f83546c-01a9-4156-8a97-44c446245d3e:22507:1862022967056128:192.168.202.146@tcp:101:ls.0 00010000:00010000:0.0:1775764854.411567:0:22507:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421ca16 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc4c7 expref: -99 pid: 22507 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.411584:0:22507:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421ca16 lrc: 4/1,0 mode: --/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc4c7 expref: -99 pid: 22507 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.411616:0:22507:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421ca16 lrc: 4/1,0 mode: --/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc4c7 expref: -99 pid: 22507 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.411629:0:22507:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421ca16 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc4c7 expref: -99 pid: 22507 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.411640:0:22507:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421ca16 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc4c7 expref: -99 pid: 22507 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.411652:0:22507:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421ca16 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc4c7 expref: -99 pid: 22507 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.411660:0:22507:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421ca16 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc4c7 expref: -99 pid: 22507 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.411668:0:22507:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421ca16 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc4c7 expref: -99 pid: 22507 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1775764854.411679:0:22507:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f9970a77b80 x1862022967056128/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/1136 e 0 to 0 dl 1775764870 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'ls.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1775764854.411690:0:22507:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421ca16 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc4c7 expref: -99 pid: 22507 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:0.0:1775764854.411703:0:22507:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421ca16 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc4c7 expref: -99 pid: 22507 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.411713:0:22507:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/3808 (0 0) 00000080:00200000:0.0:1775764854.412419:0:22507:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0xedf:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1775764854.412516:0:22507:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0xee0:0x0](ffff8f9952eb7448): rc = 0 00000080:00200000:0.0:1775764854.412542:0:22507:0:(file.c:6766:ll_layout_conf()) [0x200000401:0xee0:0x0]: layout version change: 8 -> 8 00000080:00010000:0.0:1775764854.412553:0:22507:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xee0:0x0](ffff8f9952eb7448) for lock 0x36f9bdc9e421ca16 00000080:00010000:0.0:1775764854.412564:0:22507:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xee0:0x0](ffff8f9952eb7448) 00000080:00010000:0.0:1775764854.412568:0:22507:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421ca16 from it ffffafbac19c7b20 00010000:00010000:0.0:1775764854.412573:0:22507:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421ca16 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fc4c7 expref: -99 pid: 22507 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764854.412583:0:22507:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421ca16 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fc4c7 expref: -99 pid: 22507 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200008:0.0:1775764854.412618:0:22507:0:(file.c:6226:ll_getattr_dentry()) START file f11a.sanity-pfl:[0x200000401:0xee0:0x0](ffff8f9952eb7448), request_mask 606, flags 0, foreign 0 00000080:00200000:0.0:1775764854.412624:0:22507:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xee0:0x0](ffff8f9952eb7448),name=f11a.sanity-pfl 00000002:00010000:0.0:1775764854.412644:0:22507:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xee0:0x0]) in obj [0x200000401:0xedf:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764854.412654:0:22507:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421ca16 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fc4c7 expref: -99 pid: 22507 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764854.412665:0:22507:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421ca16 lrc: 2/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fc4c7 expref: -99 pid: 22507 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1775764854.412673:0:22507:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0xedf:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764854.412677:0:22507:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xee0:0x0](ffff8f9952eb7448) 00000080:00010000:0.0:1775764854.412680:0:22507:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xee0:0x0](ffff8f9952eb7448) for lock 0x36f9bdc9e421ca16 00000080:00010000:0.0:1775764854.412684:0:22507:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421ca16 from it ffffafbac19c7cb8 00010000:00010000:0.0:1775764854.412687:0:22507:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421ca16 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fc4c7 expref: -99 pid: 22507 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764854.412721:0:22507:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99821cd000/0x36f9bdc9e421ca16 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xee0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fc4c7 expref: -99 pid: 22507 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1775764854.412921:0:22507:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xee0:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1775764854.412931:0:22507:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000401:0xee0:0x0] 00000008:00010000:0.0:1775764854.412976:0:22507:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1775764854.412980:0:22507:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f255c30, osc lock ffff8f996686e4c0, flags 0x400001002 00000008:00010000:0.0:1775764854.413000:0:22507:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1775764854.413002:0:22507:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f255c70, osc lock ffff8f996686ed10, flags 0x400001002 00000008:00010000:0.0:1775764854.413019:0:22507:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1775764854.413021:0:22507:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f255cb0, osc lock ffff8f996686fb50, flags 0x400001002 00000008:00010000:0.0:1775764854.413038:0:22507:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1775764854.413040:0:22507:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8f994f255cf0, osc lock ffff8f996686f1d0, flags 0x400001002 00010000:00010000:0.0:1775764854.413057:0:22507:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:0.0:1775764854.413071:0:22507:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 2/0,1 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000020:00200000:0.0:1775764854.413082:0:22507:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1775764854.413085:0:22507:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 5/0,1 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:0.0:1775764854.413093:0:22507:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 4/0,0 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:0.0:1775764854.413108:0:22507:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:0.0:1775764854.413131:0:22507:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (1048576 2097151) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 2/0,1 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000020:00200000:0.0:1775764854.413138:0:22507:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1775764854.413140:0:22507:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 5/0,1 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:0.0:1775764854.413147:0:22507:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 4/0,0 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:0.0:1775764854.413160:0:22507:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:0.0:1775764854.413168:0:22507:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (2097152 3145727) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 3/0,1 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00000020:00200000:0.0:1775764854.413176:0:22507:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1775764854.413179:0:22507:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 6/0,1 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:0.0:1775764854.413187:0:22507:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 5/0,0 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:0.0:1775764854.413345:0:22507:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000587dd0e8/0x36f9bdc9e421ca0f lrc: 3/0,1 mode: PW/PW res: [0x240000403:0x704:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc4ab expref: -99 pid: 22506 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764854.413368:0:22507:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (3145728 4194303) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000587dd0e8/0x36f9bdc9e421ca0f lrc: 3/0,1 mode: PW/PW res: [0x240000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc4ab expref: -99 pid: 22506 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:0.0:1775764854.413377:0:22507:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1775764854.413380:0:22507:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000587dd0e8/0x36f9bdc9e421ca0f lrc: 6/0,1 mode: PW/PW res: [0x240000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x810020000000000 nid: local remote: 0x2aac6a50e5fc4ab expref: -99 pid: 22506 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1775764854.413387:0:22507:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000587dd0e8/0x36f9bdc9e421ca0f lrc: 5/0,0 mode: PW/PW res: [0x240000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc4ab expref: -99 pid: 22506 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:0.0:1775764854.413396:0:22507:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1775764854.413546:0:22507:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000401:0xee0:0x0] updating i_size 3145729 i_blocks 4 00000020:00010000:0.0:1775764854.413551:0:22507:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8f9970b88490 (ffff8f9966dc2ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764854.413555:0:22507:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f9970b88490 (ffff8f9966dc2ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764854.413558:0:22507:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f255c30 (ffff8f9966dc27e8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764854.413563:0:22507:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f255c70 (ffff8f9966dc25a8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764854.413566:0:22507:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f255cb0 (ffff8f9966dc2cf8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764854.413569:0:22507:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8f994f255cf0 (ffff8f9966dc22d8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1775764854.413572:0:22507:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f9970b88490 (ffff8f9966dc2ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764854.413574:0:22507:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f255c30 (ffff8f9966dc2ab8/1) at cl_lock_fini():72 00000020:00010000:0.0:1775764854.413579:0:22507:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f255c70 (ffff8f9966dc2ab8/1) at cl_lock_fini():72 00000020:00010000:0.0:1775764854.413582:0:22507:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f255cb0 (ffff8f9966dc2ab8/1) at cl_lock_fini():72 00000020:00010000:0.0:1775764854.413585:0:22507:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8f994f255cf0 (ffff8f9966dc2ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764854.413628:0:22507:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xee0:0x0] ignore/verify layout 0/1, layout version 8 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764854.413702:0:22507:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:0.0:1775764854.413728:0:22507:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764854.413770:0:22507:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 33 lru_type: 1 00010000:00010000:0.0:1775764854.413780:0:22507:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 33 lru_type: 1 00000080:00200000:0.0:1775764854.413787:0:22507:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764854.413792:0:22507:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:0.0:1775764854.413796:0:22507:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:0.0:1775764854.413800:0:22507:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac19c7bf8 00010000:00010000:0.0:1775764854.413803:0:22507:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 33 lru_type: 1 00010000:00010000:0.0:1775764854.413811:0:22507:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 34 lru_type: 1 00000080:00200000:0.0:1775764854.413820:0:22507:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:0.0:1775764854.413826:0:22507:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d11a.sanity-pfl, flags=24 00000080:00200000:0.0:1775764854.413831:0:22507:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764854.413834:0:22507:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f11a.sanity-pfl, flags=8 00000080:00200000:0.0:1775764854.413869:0:22507:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:0.0:1775764854.413876:0:22507:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764854.413890:0:22507:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 34 lru_type: 1 00010000:00010000:0.0:1775764854.413899:0:22507:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 34 lru_type: 1 00000080:00200000:0.0:1775764854.413920:0:22507:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764854.413924:0:22507:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:0.0:1775764854.413926:0:22507:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:0.0:1775764854.413930:0:22507:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac19c7bf8 00010000:00010000:0.0:1775764854.413933:0:22507:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 34 lru_type: 1 00010000:00010000:0.0:1775764854.413941:0:22507:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 35 lru_type: 1 00000080:00200000:0.0:1775764854.413948:0:22507:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:0.0:1775764854.413951:0:22507:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d11a.sanity-pfl, flags=25 00000080:00200000:0.0:1775764854.413954:0:22507:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764854.413957:0:22507:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f11a.sanity-pfl, flags=9 00000008:00010000:3.0:1775764854.422812:0:2381:0:(osc_cache.c:901:osc_extent_finish()) ### extent: 00000000bb37d7a5 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00000008:00010000:3.0:1775764854.423034:0:2381:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000bb37d7a5 ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00000100:00100000:3.0:1775764854.423074:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967056000:192.168.202.146@tcp:4:rwv.0 00000080:00200000:0.0:1775764854.454609:0:22509:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:0.0:1775764854.454635:0:22509:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764854.454651:0:22509:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 35 lru_type: 1 00010000:00010000:0.0:1775764854.454667:0:22509:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 35 lru_type: 1 00000080:00200000:0.0:1775764854.454675:0:22509:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764854.454684:0:22509:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:0.0:1775764854.454687:0:22509:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:0.0:1775764854.454693:0:22509:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac1c5bbf8 00010000:00010000:0.0:1775764854.454697:0:22509:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 35 lru_type: 1 00010000:00010000:0.0:1775764854.454725:0:22509:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 36 lru_type: 1 00000080:00200000:0.0:1775764854.457590:0:22509:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:0.0:1775764854.457607:0:22509:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d11a.sanity-pfl, flags=27 00000080:00200000:2.0:1775764854.459910:0:22509:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88), inode mode 41ed mask 1 00000080:00200000:2.0:1775764854.459930:0:22509:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f11a.sanity-pfl, flags=267 00000080:00200000:2.0:1775764854.460025:0:22509:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:2.0:1775764854.460653:0:22509:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764854.460674:0:22509:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 36 lru_type: 1 00010000:00010000:2.0:1775764854.460689:0:22509:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 36 lru_type: 1 00000080:00200000:2.0:1775764854.460697:0:22509:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764854.460706:0:22509:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:2.0:1775764854.460709:0:22509:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:2.0:1775764854.460715:0:22509:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac1c5bbf8 00010000:00010000:2.0:1775764854.460718:0:22509:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 36 lru_type: 1 00010000:00010000:2.0:1775764854.460852:0:22509:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 37 lru_type: 1 00000080:00200000:2.0:1775764854.460869:0:22509:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:2.0:1775764854.460876:0:22509:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d11a.sanity-pfl, flags=267 00000080:00200000:2.0:1775764854.460883:0:22509:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88), inode mode 41ed mask 44 00000080:00200008:2.0:1775764854.460891:0:22509:0:(file.c:1057:ll_file_open()) START file d11a.sanity-pfl:[0x200000401:0xedf:0x0](ffff8f9952eedb88), flags 304000 00000080:00200080:2.0:1775764854.460934:0:22509:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88) cmd=c0086915 arg=24463d0 00000100:00100000:2.0:1775764854.461002:0:22509:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22509:1862022967056640:192.168.202.146@tcp:34:lfs.0 00000100:00100000:2.0:1775764854.461148:0:22509:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9952af3180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764854.467901:0:22509:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22509:1862022967056640:192.168.202.146@tcp:34:lfs.0 00000080:00200008:2.0:1775764854.468171:0:22509:0:(file.c:447:ll_file_release()) START file d11a.sanity-pfl:[0x200000401:0xedf:0x0](ffff8f9952eedb88), flags 304000 00010000:00010000:2.0:1775764854.468201:0:22509:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640600/0x36f9bdc9e421c967 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc220 expref: -99 pid: 22475 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00000080:00200000:0.0:1775764854.494384:0:22513:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:0.0:1775764854.494819:0:22513:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764854.494838:0:22513:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 37 lru_type: 1 00010000:00010000:0.0:1775764854.494865:0:22513:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 37 lru_type: 1 00000080:00200000:0.0:1775764854.494874:0:22513:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764854.494882:0:22513:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:0.0:1775764854.494886:0:22513:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:0.0:1775764854.494892:0:22513:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac1473bf8 00010000:00010000:0.0:1775764854.494896:0:22513:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 37 lru_type: 1 00010000:00010000:0.0:1775764854.494904:0:22513:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 38 lru_type: 1 00000080:00200000:0.0:1775764854.494919:0:22513:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:0.0:1775764854.494929:0:22513:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d11a.sanity-pfl, flags=27 00000080:00200000:0.0:1775764854.494936:0:22513:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88), inode mode 41ed mask 1 00000080:00200000:0.0:1775764854.494940:0:22513:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f11a.sanity-pfl, flags=267 00000080:00200000:0.0:1775764854.495034:0:22513:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:0.0:1775764854.495041:0:22513:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1775764854.495048:0:22513:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 38 lru_type: 1 00010000:00010000:0.0:1775764854.495057:0:22513:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 38 lru_type: 1 00000080:00200000:0.0:1775764854.495064:0:22513:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1775764854.495068:0:22513:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:0.0:1775764854.495070:0:22513:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:0.0:1775764854.495074:0:22513:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac1473bf8 00010000:00010000:0.0:1775764854.495077:0:22513:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 38 lru_type: 1 00010000:00010000:0.0:1775764854.495084:0:22513:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 39 lru_type: 1 00000080:00200000:0.0:1775764854.495093:0:22513:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:0.0:1775764854.495096:0:22513:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d11a.sanity-pfl, flags=267 00000080:00200000:0.0:1775764854.495888:0:22513:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88), inode mode 41ed mask 44 00000080:00200008:0.0:1775764854.495903:0:22513:0:(file.c:1057:ll_file_open()) START file d11a.sanity-pfl:[0x200000401:0xedf:0x0](ffff8f9952eedb88), flags 304000 00000080:00200080:0.0:1775764854.495946:0:22513:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000401:0xedf:0x0](ffff8f9952eedb88) cmd=c0086915 arg=dec3d0 00000100:00100000:0.0:1775764854.496018:0:22513:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77b80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22513:1862022967056768:192.168.202.146@tcp:34:lfs.0 00000100:00100000:0.0:1775764854.496595:0:22513:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9960102d80 going to sleep for 11 seconds 00000100:00100000:2.0:1775764854.504832:0:22513:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77b80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:22513:1862022967056768:192.168.202.146@tcp:34:lfs.0 00000080:00200008:2.0:1775764854.505035:0:22513:0:(file.c:447:ll_file_release()) START file d11a.sanity-pfl:[0x200000401:0xedf:0x0](ffff8f9952eedb88), flags 304000 00010000:00010000:2.0:1775764854.505065:0:22513:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640600/0x36f9bdc9e421c967 lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xedf:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc220 expref: -99 pid: 22475 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 0 00010000:00010000:1.0:1775764855.368462:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764856.393271:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764857.413989:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764857.546571:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:884ae1b0-72f8-4a88-a129-db99cded7efc:2381:1862022967056896:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764857.546686:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967057152:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764857.546716:0:2381:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1775764857.546721:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967057024:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764857.546772:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967057280:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764857.554826:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967057024:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764857.554841:0:2381:0:(client.c:3082:ptlrpc_free_committed()) lustre-OST0001-osc-ffff8f99523ee800: committing for last_committed 4294972366 gen 6 00000100:00100000:3.0:1775764857.554952:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967057280:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764857.554965:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:884ae1b0-72f8-4a88-a129-db99cded7efc:2381:1862022967056896:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764857.554973:0:2381:0:(client.c:3082:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8f99523ee800: committing for last_committed 4294971567 gen 5 00000100:00100000:3.0:1775764857.555003:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967057152:192.168.202.146@tcp:400:kworker.0 00010000:00010000:2.0:1775764858.448860:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764859.467627:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764860.485920:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764861.510367:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:3.0:1775764861.578446:0:23070:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-pfl test 11b: truncate file set file size correctly ========================================================== 16:00:59 (1775764859) 00010000:00010000:2.0:1775764862.541205:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764862.666918:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:884ae1b0-72f8-4a88-a129-db99cded7efc:2381:1862022967057408:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764862.667034:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967057664:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764862.667069:0:2381:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1775764862.667076:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967057536:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764862.667149:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967057792:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764862.684429:0:2381:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8f99523ee800: committing for last_committed 8589934634 gen 1 00000100:00100000:3.0:1775764862.684535:0:2381:0:(client.c:3131:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff8f994f2e5c00 x1862022967047680/t8589934619(8589934619) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 864/608 e 0 to 0 dl 1775764869 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:3.0:1775764862.684900:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967057536:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764862.684922:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967057792:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764862.684933:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:884ae1b0-72f8-4a88-a129-db99cded7efc:2381:1862022967057408:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764862.684946:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967057664:192.168.202.146@tcp:400:kworker.0 00000001:02000400:1.0:1775764863.519628:0:23219:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: SKIP: sanity-pfl test_11b needs >= 4 OSTs 00010000:00010000:0.0:1775764863.562112:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764864.584485:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764865.611863:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:0.0:1775764865.688260:0:23371:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-pfl test 12: Verify ost list specification ===== 16:01:03 (1775764863) 00010000:00010000:3.0:1775764866.634339:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1775764867.335009:0:23508:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: SKIP: sanity-pfl test_12 needs >= 3 OSTs 00010000:00010000:0.0:1775764867.656914:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764867.788196:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:884ae1b0-72f8-4a88-a129-db99cded7efc:2381:1862022967057920:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764867.788357:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967058176:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764867.788398:0:2381:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1775764867.788405:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967058048:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764867.788433:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967058304:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764867.809215:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967058048:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764867.809240:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967058304:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764867.809250:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:884ae1b0-72f8-4a88-a129-db99cded7efc:2381:1862022967057920:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764867.809260:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967058176:192.168.202.146@tcp:400:kworker.0 00010000:00010000:1.0:1775764868.679098:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1775764869.414506:0:23660:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-pfl test 13: shouldn't reprocess granted resent request ========================================================== 16:01:07 (1775764867) 00010000:00010000:3.0:1775764869.702120:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764870.730735:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:0.0:1775764871.436445:0:23796:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: SKIP: sanity-pfl test_13 needs >= 8 OSTs 00010000:00010000:3.0:1775764871.753565:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764872.773921:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764872.905900:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2380:1862022967058432:192.168.202.146@tcp:400:kworker.0 00000100:00100000:1.0:1775764872.906069:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967058688:192.168.202.146@tcp:400:kworker.0 00000100:00100000:1.0:1775764872.906111:0:2380:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1775764872.906117:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967058560:192.168.202.146@tcp:400:kworker.0 00000100:00100000:1.0:1775764872.906148:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967058816:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764872.932684:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967058560:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764872.932708:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967058816:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764872.932719:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2380:1862022967058432:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764872.932729:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967058688:192.168.202.146@tcp:400:kworker.0 00000001:02000400:3.0:1775764873.453641:0:23949:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-pfl test 14: Verify setstripe poolname/stripe_count/stripe_size inheritance ========================================================== 16:01:11 (1775764871) 00010000:00010000:2.0:1775764873.798766:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1775764873.899725:0:23816:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:2.0:1775764873.899883:0:23816:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764873.899902:0:23816:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 39 lru_type: 1 00010000:00010000:2.0:1775764873.899917:0:23816:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 39 lru_type: 1 00000080:00200000:2.0:1775764873.899926:0:23816:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764873.899937:0:23816:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:2.0:1775764873.899941:0:23816:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:2.0:1775764873.899997:0:23816:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac1827b58 00010000:00010000:2.0:1775764873.900001:0:23816:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 39 lru_type: 1 00010000:00010000:2.0:1775764873.900011:0:23816:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 40 lru_type: 1 00000080:00200000:2.0:1775764873.900028:0:23816:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:3.0:1775764873.904874:0:23816:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=d14.sanity-pfl, dir=[0x200000007:0x1:0x0](ffff8f9952d121c8), flags=9 00000080:00200000:3.0:1775764873.904888:0:23816:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=d14.sanity-pfl, dir=[0x200000007:0x1:0x0](ffff8f9952d121c8), intent=getattr 00000002:00010000:3.0:1775764873.904915:0:23816:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: d14.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764873.904978:0:23816:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421ca1d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 23816 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764873.904989:0:23816:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421ca1d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 23816 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764873.904999:0:23816:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421ca1d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 23816 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764873.905029:0:23816:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job bash:3f83546c-01a9-4156-8a97-44c446245d3e:23816:1862022967058944:192.168.202.146@tcp:101:bash.0 00000100:00100000:3.0:1775764873.905156:0:23816:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9945c06b40 going to sleep for 11 seconds 00000100:00100000:3.0:1775764873.918828:0:23816:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job bash:3f83546c-01a9-4156-8a97-44c446245d3e:23816:1862022967058944:192.168.202.146@tcp:101:bash.0 00010000:00010000:3.0:1775764873.918852:0:23816:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421ca1d lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 23816 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764873.918867:0:23816:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421ca1d lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 23816 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764873.918875:0:23816:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421ca1d lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 23816 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764873.918910:0:23816:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421ca1d lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 23816 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764873.918918:0:23816:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421ca1d lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 23816 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764873.918924:0:23816:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99435d4e00 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:3.0:1775764873.918928:0:23816:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421ca1d lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 23816 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764873.918935:0:23816:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421ca1d lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 23816 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764873.918942:0:23816:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421ca1d lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 23816 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764873.918950:0:23816:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1775764873.918955:0:23816:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421ca1d lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 23816 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764873.918961:0:23816:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421ca1d lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 23816 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764873.918979:0:23816:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=7, status=0 req@ffff8f9970eddc00 x1862022967058944/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/584 e 0 to 0 dl 1775764889 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'bash.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1775764873.919065:0:23816:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:3.0:1775764873.919092:0:23816:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 40 lru_type: 1 00010000:00010000:3.0:1775764873.919104:0:23816:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 40 lru_type: 1 00000080:00010000:3.0:1775764873.919128:0:23816:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac1827930 00010000:00010000:3.0:1775764873.919132:0:23816:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 40 lru_type: 1 00010000:00010000:3.0:1775764873.919139:0:23816:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 41 lru_type: 1 00000080:00200008:3.0:1775764873.919312:0:23816:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8f9952d121c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1775764873.919319:0:23816:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:3.0:1775764873.919331:0:23816:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764873.919339:0:23816:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 41 lru_type: 1 00010000:00010000:3.0:1775764873.919346:0:23816:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 41 lru_type: 1 00000080:00200000:3.0:1775764873.919352:0:23816:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764873.919357:0:23816:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:3.0:1775764873.919360:0:23816:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:3.0:1775764873.919365:0:23816:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac1827cb8 00010000:00010000:3.0:1775764873.919367:0:23816:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 41 lru_type: 1 00010000:00010000:3.0:1775764873.919374:0:23816:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 42 lru_type: 1 00000080:00200000:1.0:1775764873.942175:0:23965:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764873.942204:0:23965:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764873.942226:0:23965:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 42 lru_type: 1 00010000:00010000:1.0:1775764873.942243:0:23965:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 42 lru_type: 1 00000080:00200000:1.0:1775764873.942251:0:23965:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764873.942260:0:23965:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764873.942264:0:23965:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:1.0:1775764873.942271:0:23965:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac1cebbe0 00010000:00010000:1.0:1775764873.942275:0:23965:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 42 lru_type: 1 00010000:00010000:1.0:1775764873.942283:0:23965:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 43 lru_type: 1 00000080:00200000:1.0:1775764873.942347:0:23965:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764873.942357:0:23965:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d14.sanity-pfl, flags=1536 00000080:00200000:1.0:1775764873.942363:0:23965:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764873.942370:0:23965:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764873.942377:0:23965:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 43 lru_type: 1 00010000:00010000:1.0:1775764873.942385:0:23965:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 43 lru_type: 1 00000080:00200000:1.0:1775764873.942390:0:23965:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764873.942394:0:23965:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764873.942396:0:23965:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421c921 00000080:00010000:1.0:1775764873.942400:0:23965:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421c921 from it ffffafbac1cebdc8 00010000:00010000:1.0:1775764873.942403:0:23965:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 43 lru_type: 1 00010000:00010000:1.0:1775764873.942410:0:23965:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 44 lru_type: 1 00000080:00200000:1.0:1775764873.942417:0:23965:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 3 00000080:00200000:1.0:1775764873.942423:0:23965:0:(namei.c:2216:do_mkdir()) VFS Op:name=d14.sanity-pfl, dir=[0x200000007:0x1:0x0](ffff8f9952d121c8) 00010000:00010000:1.0:1775764873.942455:0:23965:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 44 lru_type: 1 00010000:00010000:1.0:1775764873.942509:0:23965:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 44 lru_type: 1 00010000:00010000:1.0:1775764873.942536:0:23965:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764873.942538:0:23965:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f9970640400/0x36f9bdc9e421c921 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fc132 expref: -99 pid: 22462 timeout: 0 lvb_type: 0 lru_score: 44 lru_type: 1 00000020:00100000:1.0:1775764873.942565:0:23965:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1775764873.942579:0:23965:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e5880 x1862022967059200/t0(0) o36->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 560/1000 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764873.942603:0:23965:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5880 pname:cluuid:pid:xid:nid:opc:job mkdir:3f83546c-01a9-4156-8a97-44c446245d3e:23965:1862022967059200:192.168.202.146@tcp:36:mkdir.0 00000100:00100000:1.0:1775764873.943612:0:23965:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560840 going to sleep for 11 seconds 00000100:00100000:1.0:1775764873.953674:0:23965:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5880 pname:cluuid:pid:xid:nid:opc:job mkdir:3f83546c-01a9-4156-8a97-44c446245d3e:23965:1862022967059200:192.168.202.146@tcp:36:mkdir.0 00000080:00200000:1.0:1775764873.953800:0:23965:0:(llite_lib.c:3177:ll_read_inode2()) VFS Op:inode=[0x200000401:0xee2:0x0](ffff8f9952e3ba88) 00000080:00200000:1.0:1775764873.953807:0:23965:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0xee2:0x0], updating i_size 512 00000080:00200000:2.0:1775764873.967722:0:23966:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:2.0:1775764873.969903:0:23966:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764873.969982:0:23966:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 23966 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764873.970004:0:23966:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 23966 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764873.970016:0:23966:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 23966 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764873.970220:0:23966:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:23966:1862022967059328:192.168.202.146@tcp:101:rm.0 00000100:00100000:2.0:1775764873.970437:0:23966:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9966b463c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764873.980494:0:23966:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:23966:1862022967059328:192.168.202.146@tcp:101:rm.0 00010000:00010000:2.0:1775764873.980525:0:23966:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764873.980536:0:23966:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764873.980549:0:23966:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764873.980560:0:23966:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764873.980573:0:23966:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764873.980581:0:23966:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764873.980587:0:23966:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764873.980598:0:23966:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f99703d3800 x1862022967059328/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764889 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764873.980675:0:23966:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764873.980691:0:23966:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1775764873.980697:0:23966:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764873.980727:0:23966:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 8704 00000080:00010000:2.0:1775764873.980732:0:23966:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421ca24 00000080:00010000:2.0:1775764873.980766:0:23966:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:2.0:1775764873.980770:0:23966:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421ca24 from it ffffafbac0da3b58 00010000:00010000:2.0:1775764873.980773:0:23966:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764873.980782:0:23966:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764873.980809:0:23966:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:2.0:1775764873.980836:0:23966:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=d14.sanity-pfl, dir=[0x200000007:0x1:0x0](ffff8f9952d121c8), flags=24 00000080:00200000:2.0:1775764873.980842:0:23966:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=d14.sanity-pfl, dir=[0x200000007:0x1:0x0](ffff8f9952d121c8), intent=0 00000002:00010000:2.0:1775764873.980861:0:23966:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: d14.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: lookup flags 0 00010000:00010000:2.0:1775764873.980904:0:23966:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421ca2b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 23966 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764873.980912:0:23966:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421ca2b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 23966 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764873.980922:0:23966:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421ca2b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 23966 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764873.980966:0:23966:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:23966:1862022967059456:192.168.202.146@tcp:101:rm.0 00000100:00100000:0.0:1775764873.981394:0:23966:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9966b463c0 going to sleep for 11 seconds 00000100:00100000:3.0:1775764873.985658:0:23966:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:23966:1862022967059456:192.168.202.146@tcp:101:rm.0 00010000:00010000:3.0:1775764873.985786:0:23966:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421ca2b lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc53e expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764873.985809:0:23966:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421ca2b lrc: 4/1,0 mode: --/PR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc53e expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764873.985818:0:23966:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421ca2b lrc: 4/1,0 mode: --/PR res: [0x200000401:0xee2:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc53e expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764873.985830:0:23966:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421ca2b lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xee2:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc53e expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764873.985841:0:23966:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421ca2b lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xee2:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc53e expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764873.985854:0:23966:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421ca2b lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xee2:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc53e expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764873.985863:0:23966:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421ca2b lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xee2:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc53e expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764873.985870:0:23966:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421ca2b lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xee2:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc53e expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764873.985880:0:23966:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=b, status=0 req@ffff8f99703d3800 x1862022967059456/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/584 e 0 to 0 dl 1775764889 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1775764873.985892:0:23966:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421ca2b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xee2:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc53e expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764873.985923:0:23966:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/3810 (0 0) 00000080:00200000:3.0:1775764873.985932:0:23966:0:(namei.c:943:ll_intent_lock()) intent lock 16 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:3.0:1775764873.985959:0:23966:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0xee2:0x0], updating i_size 512 00000080:00200000:3.0:1775764873.985961:0:23966:0:(namei.c:147:ll_iget()) got inode: [0x200000401:0xee2:0x0](ffff8f9952e3ba88): rc = 0 00000080:00010000:3.0:1775764873.985966:0:23966:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xee2:0x0](ffff8f9952e3ba88) for lock 0x36f9bdc9e421ca2b 00000080:00010000:3.0:1775764873.985979:0:23966:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xee2:0x0](ffff8f9952e3ba88) 00000080:00010000:3.0:1775764873.985984:0:23966:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421ca2b from it ffffafbac0da3a28 00010000:00010000:3.0:1775764873.985998:0:23966:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421ca2b lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xee2:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc53e expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764873.986007:0:23966:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421ca2b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xee2:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc53e expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764873.986039:0:23966:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764873.986057:0:23966:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=f14.sanity-pfl, dir=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), flags=8 00000080:00200000:3.0:1775764873.986061:0:23966:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=f14.sanity-pfl, dir=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), intent=getattr 00000002:00010000:3.0:1775764873.986082:0:23966:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f14.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000401:0xee2:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764873.986125:0:23966:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421ca32 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 23966 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764873.986132:0:23966:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421ca32 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 23966 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764873.986143:0:23966:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421ca32 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 23966 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764873.986165:0:23966:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:23966:1862022967059584:192.168.202.146@tcp:101:rm.0 00000100:00100000:3.0:1775764873.986371:0:23966:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9945c06b40 going to sleep for 11 seconds 00000100:00100000:3.0:1775764874.019306:0:23966:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:23966:1862022967059584:192.168.202.146@tcp:101:rm.0 00010000:00010000:3.0:1775764874.019331:0:23966:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421ca32 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 23966 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764874.019345:0:23966:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421ca32 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 23966 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764874.019352:0:23966:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421ca32 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 23966 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764874.019358:0:23966:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421ca32 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 23966 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764874.019367:0:23966:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421ca32 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 23966 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764874.019373:0:23966:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99435d4e00 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:3.0:1775764874.019378:0:23966:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421ca32 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 23966 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764874.019384:0:23966:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421ca32 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 23966 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764874.019390:0:23966:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421ca32 lrc: 5/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 23966 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764874.019398:0:23966:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1775764874.019404:0:23966:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421ca32 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 23966 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764874.019456:0:23966:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421ca32 lrc: 0/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 23966 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764874.019475:0:23966:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=7, status=0 req@ffff8f9970eddc00 x1862022967059584/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/584 e 0 to 0 dl 1775764889 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1775764874.019490:0:23966:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0xee2:0x0] suppgids 0 -1: rc 0 00010000:00010000:3.0:1775764874.019551:0:23966:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/3810 (0 0) 00000080:00200000:3.0:1775764874.019636:0:23966:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:3.0:1775764874.019650:0:23966:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764874.019660:0:23966:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764874.019684:0:23966:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764874.019691:0:23966:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764874.019697:0:23966:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:3.0:1775764874.019700:0:23966:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421ca24 00000080:00010000:3.0:1775764874.019705:0:23966:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421ca24 from it ffffafbac0da3c10 00010000:00010000:3.0:1775764874.019708:0:23966:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764874.019716:0:23966:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764874.019730:0:23966:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:3.0:1775764874.019771:0:23966:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d14.sanity-pfl, flags=24 00000080:00200000:3.0:1775764874.019792:0:23966:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764874.019810:0:23966:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=f14.sanity-pfl, dir=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), flags=0 00000080:00200000:3.0:1775764874.019814:0:23966:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=f14.sanity-pfl, dir=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), intent=getattr 00000002:00010000:3.0:1775764874.019826:0:23966:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f14.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000401:0xee2:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764874.019869:0:23966:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5c00/0x36f9bdc9e421ca39 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 23966 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764874.019876:0:23966:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5c00/0x36f9bdc9e421ca39 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 23966 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764874.019886:0:23966:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5c00/0x36f9bdc9e421ca39 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 23966 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764874.019909:0:23966:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:23966:1862022967059712:192.168.202.146@tcp:101:rm.0 00000100:00100000:3.0:1775764874.020554:0:23966:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9945c06b40 going to sleep for 11 seconds 00000100:00100000:3.0:1775764874.025713:0:23966:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job rm:3f83546c-01a9-4156-8a97-44c446245d3e:23966:1862022967059712:192.168.202.146@tcp:101:rm.0 00010000:00010000:3.0:1775764874.025768:0:23966:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5c00/0x36f9bdc9e421ca39 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 23966 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764874.025783:0:23966:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5c00/0x36f9bdc9e421ca39 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 23966 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764874.025791:0:23966:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5c00/0x36f9bdc9e421ca39 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 23966 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764874.025797:0:23966:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5c00/0x36f9bdc9e421ca39 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 23966 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764874.025806:0:23966:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5c00/0x36f9bdc9e421ca39 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 23966 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764874.025849:0:23966:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99435d5c00 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:3.0:1775764874.025855:0:23966:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5c00/0x36f9bdc9e421ca39 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 23966 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764874.025862:0:23966:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5c00/0x36f9bdc9e421ca39 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 23966 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764874.025870:0:23966:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5c00/0x36f9bdc9e421ca39 lrc: 5/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 23966 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764874.025878:0:23966:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1775764874.025886:0:23966:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5c00/0x36f9bdc9e421ca39 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 23966 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764874.025893:0:23966:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d5c00/0x36f9bdc9e421ca39 lrc: 0/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 23966 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764874.025911:0:23966:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=7, status=0 req@ffff8f9970eddc00 x1862022967059712/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/584 e 0 to 0 dl 1775764890 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1775764874.025926:0:23966:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0xee2:0x0] suppgids 0 -1: rc 0 00010000:00010000:3.0:1775764874.025952:0:23966:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/3810 (0 0) 00010000:00010000:1.0:1775764874.822372:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764875.191412:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764875.191473:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764875.191518:0:11131:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1775764875.191531:0:11146:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978422528 00000100:00100000:0.0:1775764875.191533:0:11131:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978422656 00000100:00100000:0.0:1775764875.191543:0:11131:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978422656 00000100:00100000:1.0:1775764875.191543:0:11146:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978422528 00000100:00100000:0.0:1775764875.191619:0:11131:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24229 00000100:00100000:0.0:1775764875.191631:0:11131:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9966bf6050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:884ae1b0-72f8-4a88-a129-db99cded7efc+4:18982:x1862022978422656:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764875.191648:0:11131:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.202.146@tcp lock: ffff8f99706eb600/0x36f9bdc9e421c879 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1400000000000 nid: local remote: 0x2aac6a50e5fb497 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764875.191729:0:11131:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764875.191735:0:11131:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9966bf6050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:884ae1b0-72f8-4a88-a129-db99cded7efc+4:18982:x1862022978422656:12345-192.168.202.146@tcp:104: Request processed in 108us (262us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764875.191803:0:11131:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24229 00010000:00010000:3.0:1775764875.192005:0:19800:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1775764875.192012:0:19800:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.146@tcp lock: ffff8f99706eb600/0x36f9bdc9e421c879 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1400000000000 nid: local remote: 0x2aac6a50e5fb497 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764875.192020:0:19800:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99706eb600 already unused, calling callback (ffffffffc1393c20) 10000000:00010000:3.0:1775764875.192024:0:19800:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.146@tcp lock: ffff8f99706eb600/0x36f9bdc9e421c879 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1400400000000 nid: local remote: 0x2aac6a50e5fb497 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764875.192052:0:19800:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.146@tcp lock: ffff8f99706eb600/0x36f9bdc9e421c879 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1408400000000 nid: local remote: 0x2aac6a50e5fb497 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:3.0:1775764875.192058:0:19800:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.146@tcp lock: ffff8f99706eb600/0x36f9bdc9e421c879 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1409400000000 nid: local remote: 0x2aac6a50e5fb497 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1775764875.192064:0:19800:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:3.0:1775764875.192100:0:19800:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: MGC192.168.202.146@tcp lock: ffff8f99706eb600/0x36f9bdc9e421c879 lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5c09400000020 nid: local remote: 0x2aac6a50e5fb497 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764875.192107:0:19800:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764875.192128:0:19800:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.146@tcp lock: ffff8f99706eb600/0x36f9bdc9e421c879 lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5c09400000020 nid: local remote: 0x2aac6a50e5fb497 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764875.192135:0:19800:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.146@tcp lock: ffff8f99706eb600/0x36f9bdc9e421c879 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5c09400000020 nid: local remote: 0x2aac6a50e5fb497 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764875.192149:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764875.192151:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764875.192153:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764875.192169:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:884ae1b0-72f8-4a88-a129-db99cded7efc:2381:1862022967059840:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764875.192192:0:11146:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24230 00000100:00100000:1.0:1775764875.192200:0:11146:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970a74380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:884ae1b0-72f8-4a88-a129-db99cded7efc+3:18982:x1862022978422528:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764875.192209:0:11146:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.202.146@tcp lock: ffff8f99706eba00/0x36f9bdc9e421c887 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0x2aac6a50e5fb4a5 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764875.192237:0:11146:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764875.192242:0:11146:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970a74380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_003:884ae1b0-72f8-4a88-a129-db99cded7efc+3:18982:x1862022978422528:12345-192.168.202.146@tcp:104: Request processed in 44us (842us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764875.192249:0:11146:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24230 00010000:00010000:2.0:1775764875.192985:0:5572:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764875.192992:0:5572:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.146@tcp lock: ffff8f99706eba00/0x36f9bdc9e421c887 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0x2aac6a50e5fb4a5 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764875.193001:0:5572:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99706eba00 already unused, calling callback (ffffffffc1393c20) 10000000:00010000:2.0:1775764875.193004:0:5572:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.146@tcp lock: ffff8f99706eba00/0x36f9bdc9e421c887 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1400400000000 nid: local remote: 0x2aac6a50e5fb4a5 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764875.193011:0:5572:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.146@tcp lock: ffff8f99706eba00/0x36f9bdc9e421c887 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1408400000000 nid: local remote: 0x2aac6a50e5fb4a5 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:2.0:1775764875.193016:0:5572:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.146@tcp lock: ffff8f99706eba00/0x36f9bdc9e421c887 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1409400000000 nid: local remote: 0x2aac6a50e5fb4a5 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1775764875.193021:0:5572:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:2.0:1775764875.193054:0:5572:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: MGC192.168.202.146@tcp lock: ffff8f99706eba00/0x36f9bdc9e421c887 lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x2aac6a50e5fb4a5 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764875.193059:0:5572:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764875.193082:0:5572:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.146@tcp lock: ffff8f99706eba00/0x36f9bdc9e421c887 lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x2aac6a50e5fb4a5 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764875.193089:0:5572:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.146@tcp lock: ffff8f99706eba00/0x36f9bdc9e421c887 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x2aac6a50e5fb4a5 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764875.193105:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764875.193107:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764875.193109:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764875.193138:0:2381:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1775764875.193149:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:884ae1b0-72f8-4a88-a129-db99cded7efc:2381:1862022967059968:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764875.201886:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:884ae1b0-72f8-4a88-a129-db99cded7efc:2381:1862022967059968:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764875.201903:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:884ae1b0-72f8-4a88-a129-db99cded7efc:2381:1862022967059840:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1775764875.847025:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1775764876.486046:0:36:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xedd:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764876.486111:0:36:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764876.486188:0:36:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xedd:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764876.486230:0:36:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xede:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764876.486249:0:36:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:1.0:1775764876.486272:0:36:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00020000:00200000:1.0:1775764876.486274:0:36:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x0 00000080:00200000:1.0:1775764876.486287:0:36:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xede:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 10000000:01000000:0.0:1775764876.805897:0:5575:0:(mgc_request.c:611:do_requeue()) updating log lustre-client 10000000:01000000:0.0:1775764876.805934:0:5575:0:(mgc_request.c:1819:mgc_process_log()) Process log lustre-client-ffff8f99523ee800 from 42 10000000:01000000:0.0:1775764876.805941:0:5575:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-client (res 0x65727473756c) 00010000:00010000:0.0:1775764876.806000:0:5575:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.146@tcp lock: ffff8f99821cd800/0x36f9bdc9e421ca40 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764876.806013:0:5575:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.146@tcp lock: ffff8f99821cd800/0x36f9bdc9e421ca40 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764876.806025:0:5575:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.146@tcp lock: ffff8f99821cd800/0x36f9bdc9e421ca40 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764876.806055:0:5575:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967060096:192.168.202.146@tcp:101:ll_cfg_requeue.0 00000100:00100000:0.0:1775764876.806195:0:5575:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f997130d180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764876.817564:0:5575:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967060096:192.168.202.146@tcp:101:ll_cfg_requeue.0 00010000:00010000:2.0:1775764876.817586:0:5575:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.202.146@tcp lock: ffff8f99821cd800/0x36f9bdc9e421ca40 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x2aac6a50e5fc5ca expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764876.817597:0:5575:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.202.146@tcp lock: ffff8f99821cd800/0x36f9bdc9e421ca40 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x2aac6a50e5fc5ca expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764876.817681:0:5575:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967060224:192.168.202.146@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1775764876.817786:0:5575:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9966b46cc0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764876.819904:0:5575:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967060224:192.168.202.146@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1775764876.819967:0:5575:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967060352:192.168.202.146@tcp:503:ll_cfg_requeue.0 00000100:00100000:2.0:1775764876.820074:0:5575:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9966b46cc0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764876.825655:0:5575:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967060352:192.168.202.146@tcp:503:ll_cfg_requeue.0 00000100:00100000:2.0:1775764876.826152:0:23977:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job llog_process_th:884ae1b0-72f8-4a88-a129-db99cded7efc:23977:1862022967060480:192.168.202.146@tcp:502:llog_process_th.0 00000100:00100000:2.0:1775764876.826256:0:23977:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9966b46840 going to sleep for 11 seconds 00000100:00100000:2.0:1775764876.830260:0:23977:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job llog_process_th:884ae1b0-72f8-4a88-a129-db99cded7efc:23977:1862022967060480:192.168.202.146@tcp:502:llog_process_th.0 00000020:01000000:2.0:1775764876.830318:0:23977:0:(obd_config.c:1888:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:2.0:1775764876.830329:0:23977:0:(obd_config.c:1468:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1775764876.830357:0:23977:0:(obd_config.c:1559:class_process_config()) marker 28 (0x1) lustre-clilov new lustre.pool1 00000020:01000000:2.0:1775764876.830365:0:23977:0:(obd_config.c:2003:class_config_llog_handler()) cmd ce020, instance name: lustre-clilov-ffff8f99523ee800 00000020:00000080:2.0:1775764876.830368:0:23977:0:(obd_config.c:1468:class_process_config()) processing cmd: ce020 00020000:01000000:2.0:1775764876.830390:0:23977:0:(lov_pool.c:418:lov_pool_new()) pool1 is pool #1 00000100:00100000:2.0:1775764876.830446:0:23977:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job llog_process_th:884ae1b0-72f8-4a88-a129-db99cded7efc:23977:1862022967060608:192.168.202.146@tcp:502:llog_process_th.0 00000100:00100000:2.0:1775764876.830733:0:23977:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9966b46300 going to sleep for 11 seconds 00000100:00100000:2.0:1775764876.834171:0:23977:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job llog_process_th:884ae1b0-72f8-4a88-a129-db99cded7efc:23977:1862022967060608:192.168.202.146@tcp:502:llog_process_th.0 00000020:01000000:2.0:1775764876.834221:0:23977:0:(obd_config.c:1888:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:2.0:1775764876.834231:0:23977:0:(obd_config.c:1468:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1775764876.834234:0:23977:0:(obd_config.c:1559:class_process_config()) marker 28 (0x2) lustre-clilov new lustre.pool1 00000040:00080000:2.0:1775764876.834239:0:23977:0:(llog.c:857:llog_process_thread()) stop processing plain [0xa:0x3:0x0] index 45 count 45 00000020:01000000:0.0:1775764876.834421:0:5575:0:(obd_config.c:2143:class_config_parse_llog()) Processed log lustre-client gen 42-44 (rc=0) 10000000:01000000:0.0:1775764876.834438:0:5575:0:(mgc_request.c:1896:mgc_process_log()) MGC192.168.202.146@tcp: configuration from log 'lustre-client' succeeded (0). 00010000:00010000:0.0:1775764876.834449:0:5575:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.146@tcp lock: ffff8f99821cd800/0x36f9bdc9e421ca40 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x2aac6a50e5fc5ca expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764876.834460:0:5575:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.202.146@tcp lock: ffff8f99821cd800/0x36f9bdc9e421ca40 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x2aac6a50e5fc5ca expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1775764876.834470:0:5575:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:01000000:0.0:1775764876.834473:0:5575:0:(mgc_request.c:1819:mgc_process_log()) Process log lustre-sptlrpc-ffff8f9942a1b380 from 1 10000000:01000000:0.0:1775764876.834476:0:5575:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00010000:00010000:0.0:1775764876.834508:0:5575:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.146@tcp lock: ffff8f99821cc200/0x36f9bdc9e421ca47 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764876.834517:0:5575:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.146@tcp lock: ffff8f99821cc200/0x36f9bdc9e421ca47 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764876.834526:0:5575:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.146@tcp lock: ffff8f99821cc200/0x36f9bdc9e421ca47 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764876.834597:0:5575:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967060736:192.168.202.146@tcp:101:ll_cfg_requeue.0 00000100:00100000:0.0:1775764876.834706:0:5575:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f997130de40 going to sleep for 11 seconds 00000100:00100000:2.0:1775764876.838344:0:5575:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967060736:192.168.202.146@tcp:101:ll_cfg_requeue.0 00010000:00010000:2.0:1775764876.838373:0:5575:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.202.146@tcp lock: ffff8f99821cc200/0x36f9bdc9e421ca47 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x2aac6a50e5fc5df expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764876.838386:0:5575:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.202.146@tcp lock: ffff8f99821cc200/0x36f9bdc9e421ca47 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x2aac6a50e5fc5df expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764876.838473:0:5575:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967060864:192.168.202.146@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1775764876.838585:0:5575:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9966b46300 going to sleep for 11 seconds 00000100:00100000:2.0:1775764876.842272:0:5575:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967060864:192.168.202.146@tcp:501:ll_cfg_requeue.0 10000000:01000000:2.0:1775764876.842319:0:5575:0:(mgc_request.c:1896:mgc_process_log()) MGC192.168.202.146@tcp: configuration from log 'lustre-sptlrpc' failed (-2). 00010000:00010000:2.0:1775764876.842329:0:5575:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.146@tcp lock: ffff8f99821cc200/0x36f9bdc9e421ca47 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x2aac6a50e5fc5df expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764876.842339:0:5575:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.202.146@tcp lock: ffff8f99821cc200/0x36f9bdc9e421ca47 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x2aac6a50e5fc5df expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764876.869895:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764877.896621:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764878.022140:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2380:1862022967060992:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764878.022247:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967061248:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764878.022279:0:2380:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:0.0:1775764878.022301:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967061120:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764878.022331:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967061376:192.168.202.146@tcp:400:kworker.0 00000100:00100000:1.0:1775764878.042736:0:2380:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8f99523ee800: committing for last_committed 8589934635 gen 1 00000100:00100000:1.0:1775764878.042795:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967061120:192.168.202.146@tcp:400:kworker.0 00000100:00100000:1.0:1775764878.042811:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967061376:192.168.202.146@tcp:400:kworker.0 00000100:00100000:1.0:1775764878.042821:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967061248:192.168.202.146@tcp:400:kworker.0 00000100:00100000:1.0:1775764878.047904:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2380:1862022967060992:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764878.383157:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764878.383197:0:2372:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764878.383244:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978428160 00000100:00100000:0.0:1775764878.383254:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978428160 00000100:00100000:0.0:1775764878.383306:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978428416 00000100:00100000:0.0:1775764878.383309:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978428416 00000100:00100000:0.0:1775764878.383332:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24231 00000100:00100000:0.0:1775764878.383342:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970a74e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:884ae1b0-72f8-4a88-a129-db99cded7efc+4:19037:x1862022978428160:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764878.383360:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.202.146@tcp lock: ffff8f99821cc200/0x36f9bdc9e421ca47 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1400000000000 nid: local remote: 0x2aac6a50e5fc5df expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764878.383443:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764878.383450:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970a74e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:884ae1b0-72f8-4a88-a129-db99cded7efc+4:19037:x1862022978428160:12345-192.168.202.146@tcp:104: Request processed in 110us (305us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764878.383457:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24231 00000100:00100000:0.0:1775764878.383474:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24232 00000100:00100000:0.0:1775764878.383519:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970a75880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:884ae1b0-72f8-4a88-a129-db99cded7efc+3:19037:x1862022978428416:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764878.383526:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.202.146@tcp lock: ffff8f99821cd800/0x36f9bdc9e421ca40 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1400000000000 nid: local remote: 0x2aac6a50e5fc5ca expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764878.383556:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:0.0:1775764878.383561:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970a75880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:884ae1b0-72f8-4a88-a129-db99cded7efc+3:19037:x1862022978428416:12345-192.168.202.146@tcp:104: Request processed in 44us (364us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764878.383567:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24232 00010000:00010000:3.0:1775764878.383683:0:19800:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1775764878.383692:0:19800:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.146@tcp lock: ffff8f99821cc200/0x36f9bdc9e421ca47 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1400000000000 nid: local remote: 0x2aac6a50e5fc5df expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764878.383701:0:19800:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99821cc200 already unused, calling callback (ffffffffc1393c20) 10000000:00010000:3.0:1775764878.383706:0:19800:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.146@tcp lock: ffff8f99821cc200/0x36f9bdc9e421ca47 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1400400000000 nid: local remote: 0x2aac6a50e5fc5df expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764878.383736:0:19800:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.146@tcp lock: ffff8f99821cc200/0x36f9bdc9e421ca47 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1408400000000 nid: local remote: 0x2aac6a50e5fc5df expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764878.383754:0:5571:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764878.383761:0:5571:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.146@tcp lock: ffff8f99821cd800/0x36f9bdc9e421ca40 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x1400000000000 nid: local remote: 0x2aac6a50e5fc5ca expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764878.383773:0:5571:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99821cd800 already unused, calling callback (ffffffffc1393c20) 10000000:00010000:2.0:1775764878.383776:0:5571:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.146@tcp lock: ffff8f99821cd800/0x36f9bdc9e421ca40 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x1400400000000 nid: local remote: 0x2aac6a50e5fc5ca expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764878.383784:0:5571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.146@tcp lock: ffff8f99821cd800/0x36f9bdc9e421ca40 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x1408400000000 nid: local remote: 0x2aac6a50e5fc5ca expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:2.0:1775764878.383790:0:5571:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.146@tcp lock: ffff8f99821cd800/0x36f9bdc9e421ca40 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x1409400000000 nid: local remote: 0x2aac6a50e5fc5ca expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1775764878.383796:0:5571:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:2.0:1775764878.383836:0:5571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: MGC192.168.202.146@tcp lock: ffff8f99821cd800/0x36f9bdc9e421ca40 lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x5c09400000020 nid: local remote: 0x2aac6a50e5fc5ca expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764878.383843:0:5571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764878.383865:0:5571:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.146@tcp lock: ffff8f99821cd800/0x36f9bdc9e421ca40 lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x5c09400000020 nid: local remote: 0x2aac6a50e5fc5ca expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764878.383872:0:5571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.146@tcp lock: ffff8f99821cd800/0x36f9bdc9e421ca40 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x5c09400000020 nid: local remote: 0x2aac6a50e5fc5ca expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764878.383886:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764878.383889:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764878.383891:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764878.383942:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:884ae1b0-72f8-4a88-a129-db99cded7efc:2381:1862022967061504:192.168.202.146@tcp:103:ldlm_bl.0 10000000:00010000:3.0:1775764878.383966:0:19800:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.146@tcp lock: ffff8f99821cc200/0x36f9bdc9e421ca47 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1409400000000 nid: local remote: 0x2aac6a50e5fc5df expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1775764878.383973:0:19800:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:3.0:1775764878.384122:0:19800:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: MGC192.168.202.146@tcp lock: ffff8f99821cc200/0x36f9bdc9e421ca47 lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x2aac6a50e5fc5df expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764878.384129:0:19800:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764878.384151:0:19800:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.146@tcp lock: ffff8f99821cc200/0x36f9bdc9e421ca47 lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x2aac6a50e5fc5df expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764878.384158:0:19800:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.146@tcp lock: ffff8f99821cc200/0x36f9bdc9e421ca47 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x2aac6a50e5fc5df expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764878.384173:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764878.384176:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764878.384179:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764878.384244:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2382:1862022967061632:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764878.387380:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:884ae1b0-72f8-4a88-a129-db99cded7efc:2381:1862022967061504:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764878.395092:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2382:1862022967061632:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1775764878.918069:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:01000000:1.0:1775764879.813865:0:5575:0:(mgc_request.c:611:do_requeue()) updating log lustre-client 10000000:01000000:1.0:1775764879.813875:0:5575:0:(mgc_request.c:1819:mgc_process_log()) Process log lustre-client-ffff8f99523ee800 from 45 10000000:01000000:1.0:1775764879.813883:0:5575:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-client (res 0x65727473756c) 00010000:00010000:1.0:1775764879.813958:0:5575:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.146@tcp lock: ffff8f996695c000/0x36f9bdc9e421ca4e lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764879.813970:0:5575:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.146@tcp lock: ffff8f996695c000/0x36f9bdc9e421ca4e lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764879.813982:0:5575:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.146@tcp lock: ffff8f996695c000/0x36f9bdc9e421ca4e lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764879.814011:0:5575:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967061760:192.168.202.146@tcp:101:ll_cfg_requeue.0 00000100:00100000:1.0:1775764879.814175:0:5575:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560240 going to sleep for 11 seconds 00000100:00100000:1.0:1775764879.824236:0:5575:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967061760:192.168.202.146@tcp:101:ll_cfg_requeue.0 00010000:00010000:1.0:1775764879.824268:0:5575:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.202.146@tcp lock: ffff8f996695c000/0x36f9bdc9e421ca4e lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x2aac6a50e5fc5ed expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764879.824282:0:5575:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.202.146@tcp lock: ffff8f996695c000/0x36f9bdc9e421ca4e lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x2aac6a50e5fc5ed expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764879.824626:0:5575:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967061888:192.168.202.146@tcp:501:ll_cfg_requeue.0 00000100:00100000:1.0:1775764879.824716:0:5575:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560240 going to sleep for 11 seconds 00000100:00100000:1.0:1775764879.831670:0:5575:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967061888:192.168.202.146@tcp:501:ll_cfg_requeue.0 00000100:00100000:1.0:1775764879.831772:0:5575:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967062016:192.168.202.146@tcp:503:ll_cfg_requeue.0 00000100:00100000:1.0:1775764879.831886:0:5575:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560240 going to sleep for 11 seconds 00000100:00100000:1.0:1775764879.844306:0:5575:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967062016:192.168.202.146@tcp:503:ll_cfg_requeue.0 00000100:00100000:1.0:1775764879.845774:0:24005:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job llog_process_th:884ae1b0-72f8-4a88-a129-db99cded7efc:24005:1862022967062144:192.168.202.146@tcp:502:llog_process_th.0 00000100:00100000:1.0:1775764879.845899:0:24005:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560540 going to sleep for 11 seconds 00000100:00100000:1.0:1775764879.854160:0:24005:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job llog_process_th:884ae1b0-72f8-4a88-a129-db99cded7efc:24005:1862022967062144:192.168.202.146@tcp:502:llog_process_th.0 00000020:01000000:1.0:1775764879.854232:0:24005:0:(obd_config.c:1888:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:1.0:1775764879.854249:0:24005:0:(obd_config.c:1468:class_process_config()) processing cmd: cf010 00000020:00000080:1.0:1775764879.854251:0:24005:0:(obd_config.c:1559:class_process_config()) marker 30 (0x1) lustre-clilov new lustre.pool2 00000020:01000000:1.0:1775764879.854259:0:24005:0:(obd_config.c:2003:class_config_llog_handler()) cmd ce020, instance name: lustre-clilov-ffff8f99523ee800 00000020:00000080:1.0:1775764879.854262:0:24005:0:(obd_config.c:1468:class_process_config()) processing cmd: ce020 00020000:01000000:1.0:1775764879.854285:0:24005:0:(lov_pool.c:418:lov_pool_new()) pool2 is pool #2 00000100:00100000:1.0:1775764879.854338:0:24005:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job llog_process_th:884ae1b0-72f8-4a88-a129-db99cded7efc:24005:1862022967062272:192.168.202.146@tcp:502:llog_process_th.0 00000100:00100000:1.0:1775764879.854443:0:24005:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560840 going to sleep for 11 seconds 00000100:00100000:1.0:1775764879.864287:0:24005:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job llog_process_th:884ae1b0-72f8-4a88-a129-db99cded7efc:24005:1862022967062272:192.168.202.146@tcp:502:llog_process_th.0 00000020:01000000:1.0:1775764879.864353:0:24005:0:(obd_config.c:1888:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:1.0:1775764879.864365:0:24005:0:(obd_config.c:1468:class_process_config()) processing cmd: cf010 00000020:00000080:1.0:1775764879.864368:0:24005:0:(obd_config.c:1559:class_process_config()) marker 30 (0x2) lustre-clilov new lustre.pool2 00000040:00080000:1.0:1775764879.864537:0:24005:0:(llog.c:857:llog_process_thread()) stop processing plain [0xa:0x3:0x0] index 48 count 48 00000020:01000000:0.0:1775764879.865189:0:5575:0:(obd_config.c:2143:class_config_parse_llog()) Processed log lustre-client gen 45-47 (rc=0) 10000000:01000000:0.0:1775764879.865221:0:5575:0:(mgc_request.c:1896:mgc_process_log()) MGC192.168.202.146@tcp: configuration from log 'lustre-client' succeeded (0). 00010000:00010000:0.0:1775764879.865234:0:5575:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.146@tcp lock: ffff8f996695c000/0x36f9bdc9e421ca4e lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x2aac6a50e5fc5ed expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764879.865246:0:5575:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.202.146@tcp lock: ffff8f996695c000/0x36f9bdc9e421ca4e lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x2aac6a50e5fc5ed expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1775764879.865257:0:5575:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:01000000:0.0:1775764879.865261:0:5575:0:(mgc_request.c:1819:mgc_process_log()) Process log lustre-sptlrpc-ffff8f9942a1b380 from 1 10000000:01000000:0.0:1775764879.865263:0:5575:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00010000:00010000:0.0:1775764879.865303:0:5575:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.146@tcp lock: ffff8f99821cce00/0x36f9bdc9e421ca55 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764879.865312:0:5575:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.146@tcp lock: ffff8f99821cce00/0x36f9bdc9e421ca55 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764879.865322:0:5575:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.146@tcp lock: ffff8f99821cce00/0x36f9bdc9e421ca55 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764879.865361:0:5575:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967062400:192.168.202.146@tcp:101:ll_cfg_requeue.0 00000100:00100000:0.0:1775764879.868657:0:5575:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f997130d900 going to sleep for 11 seconds 00000100:00100000:1.0:1775764879.885862:0:5575:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967062400:192.168.202.146@tcp:101:ll_cfg_requeue.0 00010000:00010000:1.0:1775764879.885896:0:5575:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.202.146@tcp lock: ffff8f99821cce00/0x36f9bdc9e421ca55 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x2aac6a50e5fc5f4 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764879.886782:0:5575:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.202.146@tcp lock: ffff8f99821cce00/0x36f9bdc9e421ca55 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x2aac6a50e5fc5f4 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764879.886901:0:5575:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967062528:192.168.202.146@tcp:501:ll_cfg_requeue.0 00000100:00100000:1.0:1775764879.887408:0:5575:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560840 going to sleep for 11 seconds 00000100:00100000:2.0:1775764879.894588:0:5575:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967062528:192.168.202.146@tcp:501:ll_cfg_requeue.0 10000000:01000000:2.0:1775764879.894650:0:5575:0:(mgc_request.c:1896:mgc_process_log()) MGC192.168.202.146@tcp: configuration from log 'lustre-sptlrpc' failed (-2). 00010000:00010000:2.0:1775764879.894661:0:5575:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.146@tcp lock: ffff8f99821cce00/0x36f9bdc9e421ca55 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x2aac6a50e5fc5f4 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764879.894672:0:5575:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.202.146@tcp lock: ffff8f99821cce00/0x36f9bdc9e421ca55 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x2aac6a50e5fc5f4 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764879.941961:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1775764880.661571:0:24022:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764880.661595:0:24022:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764880.661625:0:24022:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764880.661641:0:24022:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764880.661648:0:24022:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764880.661661:0:24022:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764880.661665:0:24022:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421ca24 00000080:00010000:1.0:1775764880.661673:0:24022:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421ca24 from it ffffafbac1c7bb58 00010000:00010000:1.0:1775764880.661676:0:24022:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764880.661698:0:24022:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764880.661711:0:24022:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764880.661717:0:24022:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d14.sanity-pfl, flags=24 00000080:00200000:1.0:1775764880.661722:0:24022:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764880.661730:0:24022:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=f14.sanity-pfl, dir=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), flags=8 00000080:00200000:1.0:1775764880.661733:0:24022:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=f14.sanity-pfl, dir=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), intent=getattr 00000002:00010000:1.0:1775764880.661776:0:24022:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f14.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000401:0xee2:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764880.661821:0:24022:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d000/0x36f9bdc9e421ca5c lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.661827:0:24022:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d000/0x36f9bdc9e421ca5c lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.661835:0:24022:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d000/0x36f9bdc9e421ca5c lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764880.661855:0:24022:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24022:1862022967062656:192.168.202.146@tcp:101:lfs.0 00000100:00100000:1.0:1775764880.661937:0:24022:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560240 going to sleep for 11 seconds 00000100:00100000:1.0:1775764880.665680:0:24022:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24022:1862022967062656:192.168.202.146@tcp:101:lfs.0 00010000:00010000:1.0:1775764880.665699:0:24022:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d000/0x36f9bdc9e421ca5c lrc: 4/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.665709:0:24022:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d000/0x36f9bdc9e421ca5c lrc: 4/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.665842:0:24022:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d000/0x36f9bdc9e421ca5c lrc: 4/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.665846:0:24022:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d000/0x36f9bdc9e421ca5c lrc: 3/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.665852:0:24022:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d000/0x36f9bdc9e421ca5c lrc: 4/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.665856:0:24022:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f996695d000 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:1.0:1775764880.665859:0:24022:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d000/0x36f9bdc9e421ca5c lrc: 4/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.665863:0:24022:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d000/0x36f9bdc9e421ca5c lrc: 4/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.665869:0:24022:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d000/0x36f9bdc9e421ca5c lrc: 5/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.665874:0:24022:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1775764880.665879:0:24022:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d000/0x36f9bdc9e421ca5c lrc: 3/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.665883:0:24022:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d000/0x36f9bdc9e421ca5c lrc: 0/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764880.665897:0:24022:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=7, status=0 req@ffff8f994f2e6680 x1862022967062656/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/584 e 0 to 0 dl 1775764896 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764880.665908:0:24022:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0xee2:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1775764880.665983:0:24022:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/3810 (0 0) 00000080:00200000:1.0:1775764880.666148:0:24022:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764880.666159:0:24022:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764880.666167:0:24022:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764880.666175:0:24022:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764880.666179:0:24022:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764880.666183:0:24022:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764880.666185:0:24022:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421ca24 00000080:00010000:1.0:1775764880.666189:0:24022:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421ca24 from it ffffafbac1c7bb58 00010000:00010000:1.0:1775764880.666191:0:24022:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764880.666196:0:24022:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764880.666204:0:24022:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764880.666209:0:24022:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d14.sanity-pfl, flags=29 00000080:00200000:1.0:1775764880.666213:0:24022:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764880.666221:0:24022:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=f14.sanity-pfl, dir=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), flags=13 00000080:00200000:1.0:1775764880.666224:0:24022:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=f14.sanity-pfl, dir=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), intent=getattr 00000002:00010000:1.0:1775764880.666233:0:24022:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f14.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000401:0xee2:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764880.666271:0:24022:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d200/0x36f9bdc9e421ca63 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.666279:0:24022:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d200/0x36f9bdc9e421ca63 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.666303:0:24022:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d200/0x36f9bdc9e421ca63 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764880.666323:0:24022:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24022:1862022967062784:192.168.202.146@tcp:101:lfs.0 00000100:00100000:1.0:1775764880.666533:0:24022:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560240 going to sleep for 11 seconds 00000100:00100000:1.0:1775764880.669706:0:24022:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24022:1862022967062784:192.168.202.146@tcp:101:lfs.0 00010000:00010000:1.0:1775764880.669730:0:24022:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d200/0x36f9bdc9e421ca63 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.669840:0:24022:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d200/0x36f9bdc9e421ca63 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.669846:0:24022:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d200/0x36f9bdc9e421ca63 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.669851:0:24022:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d200/0x36f9bdc9e421ca63 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.669856:0:24022:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d200/0x36f9bdc9e421ca63 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.669860:0:24022:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f996695d200 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:1.0:1775764880.669863:0:24022:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d200/0x36f9bdc9e421ca63 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.669868:0:24022:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d200/0x36f9bdc9e421ca63 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.669873:0:24022:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d200/0x36f9bdc9e421ca63 lrc: 5/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.669897:0:24022:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1775764880.669902:0:24022:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d200/0x36f9bdc9e421ca63 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.669906:0:24022:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d200/0x36f9bdc9e421ca63 lrc: 0/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764880.669919:0:24022:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=7, status=0 req@ffff8f994f2e6680 x1862022967062784/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/584 e 0 to 0 dl 1775764896 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764880.669937:0:24022:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0xee2:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1775764880.669959:0:24022:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/3810 (0 0) 00000080:00200000:1.0:1775764880.670027:0:24022:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764880.670039:0:24022:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764880.670141:0:24022:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764880.670150:0:24022:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764880.670156:0:24022:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764880.670160:0:24022:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764880.670162:0:24022:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421ca24 00000080:00010000:1.0:1775764880.670165:0:24022:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421ca24 from it ffffafbac1c7bbe8 00010000:00010000:1.0:1775764880.670167:0:24022:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764880.670173:0:24022:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764880.670193:0:24022:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764880.670199:0:24022:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d14.sanity-pfl, flags=16408 00000080:00200000:1.0:1775764880.670203:0:24022:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764880.670212:0:24022:0:(namei.c:1322:ll_lookup_nd()) VFS Op:name=f14.sanity-pfl, dir=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), flags=16392 00000080:00200000:1.0:1775764880.670215:0:24022:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=f14.sanity-pfl, dir=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), intent=getattr 00000002:00010000:1.0:1775764880.670224:0:24022:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f14.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000401:0xee2:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764880.670252:0:24022:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695cc00/0x36f9bdc9e421ca6a lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.670256:0:24022:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695cc00/0x36f9bdc9e421ca6a lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.670262:0:24022:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695cc00/0x36f9bdc9e421ca6a lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764880.670276:0:24022:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24022:1862022967062912:192.168.202.146@tcp:101:lfs.0 00000100:00100000:1.0:1775764880.670372:0:24022:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560240 going to sleep for 11 seconds 00000100:00100000:1.0:1775764880.672723:0:24022:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24022:1862022967062912:192.168.202.146@tcp:101:lfs.0 00010000:00010000:1.0:1775764880.672766:0:24022:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695cc00/0x36f9bdc9e421ca6a lrc: 4/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.672777:0:24022:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695cc00/0x36f9bdc9e421ca6a lrc: 4/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.672784:0:24022:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695cc00/0x36f9bdc9e421ca6a lrc: 4/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.672790:0:24022:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695cc00/0x36f9bdc9e421ca6a lrc: 3/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.672799:0:24022:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695cc00/0x36f9bdc9e421ca6a lrc: 4/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.672805:0:24022:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f996695cc00 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:1.0:1775764880.672809:0:24022:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695cc00/0x36f9bdc9e421ca6a lrc: 4/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.672814:0:24022:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695cc00/0x36f9bdc9e421ca6a lrc: 4/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.672818:0:24022:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695cc00/0x36f9bdc9e421ca6a lrc: 5/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.672822:0:24022:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1775764880.672827:0:24022:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695cc00/0x36f9bdc9e421ca6a lrc: 3/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.672831:0:24022:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695cc00/0x36f9bdc9e421ca6a lrc: 0/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764880.672843:0:24022:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=7, status=0 req@ffff8f994f2e6680 x1862022967062912/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 584/584 e 0 to 0 dl 1775764896 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764880.672853:0:24022:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000401:0xee2:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1775764880.672872:0:24022:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/3810 (0 0) 00000080:00200000:1.0:1775764880.672988:0:24022:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764880.672999:0:24022:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764880.673006:0:24022:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764880.673017:0:24022:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764880.673022:0:24022:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764880.673026:0:24022:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764880.673028:0:24022:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421ca24 00000080:00010000:1.0:1775764880.673031:0:24022:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421ca24 from it ffffafbac1c7bb58 00010000:00010000:1.0:1775764880.673033:0:24022:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764880.673038:0:24022:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:1.0:1775764880.673099:0:24022:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764880.673105:0:24022:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d14.sanity-pfl, flags=13 00000080:00200008:1.0:1775764880.673112:0:24022:0:(file.c:6226:ll_getattr_dentry()) START file d14.sanity-pfl:[0x200000401:0xee2:0x0](ffff8f9952e3ba88), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1775764880.673115:0:24022:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000401:0xee2:0x0](ffff8f9952e3ba88),name=d14.sanity-pfl 00000002:00010000:1.0:1775764880.673123:0:24022:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xee2:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764880.673127:0:24022:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 30 res 8589935617/3810 (0 0) 00010000:00010000:1.0:1775764880.673153:0:24022:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d400/0x36f9bdc9e421ca71 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.673157:0:24022:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d400/0x36f9bdc9e421ca71 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.673164:0:24022:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d400/0x36f9bdc9e421ca71 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764880.673181:0:24022:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24022:1862022967063040:192.168.202.146@tcp:101:lfs.0 00000100:00100000:1.0:1775764880.673242:0:24022:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560240 going to sleep for 11 seconds 00000100:00100000:1.0:1775764880.677075:0:24022:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24022:1862022967063040:192.168.202.146@tcp:101:lfs.0 00010000:00010000:1.0:1775764880.677100:0:24022:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d400/0x36f9bdc9e421ca71 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc6a3 expref: -99 pid: 24022 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.677117:0:24022:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d400/0x36f9bdc9e421ca71 lrc: 4/1,0 mode: --/PR res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc6a3 expref: -99 pid: 24022 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.677123:0:24022:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d400/0x36f9bdc9e421ca71 lrc: 4/1,0 mode: --/PR res: [0x200000401:0xee2:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc6a3 expref: -99 pid: 24022 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.677131:0:24022:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d400/0x36f9bdc9e421ca71 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xee2:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc6a3 expref: -99 pid: 24022 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.677138:0:24022:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d400/0x36f9bdc9e421ca71 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xee2:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc6a3 expref: -99 pid: 24022 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.677147:0:24022:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d400/0x36f9bdc9e421ca71 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xee2:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc6a3 expref: -99 pid: 24022 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.677152:0:24022:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d400/0x36f9bdc9e421ca71 lrc: 5/2,0 mode: PR/PR res: [0x200000401:0xee2:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc6a3 expref: -99 pid: 24022 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.677157:0:24022:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d400/0x36f9bdc9e421ca71 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xee2:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc6a3 expref: -99 pid: 24022 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764880.677168:0:24022:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8f994f2e6680 x1862022967063040/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 576/584 e 0 to 0 dl 1775764896 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764880.677177:0:24022:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d400/0x36f9bdc9e421ca71 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xee2:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc6a3 expref: -99 pid: 24022 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.677184:0:24022:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/3810 (0 0) 00000080:00200000:1.0:1775764880.677189:0:24022:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1775764880.677208:0:24022:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0xee2:0x0], updating i_size 7680 00000080:00010000:1.0:1775764880.677212:0:24022:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xee2:0x0](ffff8f9952e3ba88) for lock 0x36f9bdc9e421ca71 00000080:00010000:1.0:1775764880.677219:0:24022:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xee2:0x0](ffff8f9952e3ba88) 00000080:00010000:1.0:1775764880.677221:0:24022:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421ca71 from it ffffafbac1c7bcb8 00010000:00010000:1.0:1775764880.677223:0:24022:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d400/0x36f9bdc9e421ca71 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xee2:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc6a3 expref: -99 pid: 24022 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.677229:0:24022:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d400/0x36f9bdc9e421ca71 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xee2:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc6a3 expref: -99 pid: 24022 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:1.0:1775764880.677817:0:24022:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8f9952d121c8), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1775764880.677821:0:24022:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764880.677833:0:24022:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764880.677841:0:24022:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764880.677849:0:24022:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:1.0:1775764880.677856:0:24022:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764880.677860:0:24022:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764880.677862:0:24022:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421ca24 00000080:00010000:1.0:1775764880.677865:0:24022:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421ca24 from it ffffafbac1c7bcb8 00010000:00010000:1.0:1775764880.677867:0:24022:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764880.677871:0:24022:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1775764880.678410:0:24022:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764880.678419:0:24022:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764880.678427:0:24022:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764880.678436:0:24022:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1775764880.678442:0:24022:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764880.678447:0:24022:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764880.678449:0:24022:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421ca24 00000080:00010000:1.0:1775764880.678453:0:24022:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421ca24 from it ffffafbac1c7bbf8 00010000:00010000:1.0:1775764880.678456:0:24022:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764880.678464:0:24022:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1775764880.678473:0:24022:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764880.678479:0:24022:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d14.sanity-pfl, flags=25 00000080:00200000:1.0:1775764880.678483:0:24022:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764880.678493:0:24022:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), inode mode 41ed mask 3 00000080:00200000:1.0:1775764880.678498:0:24022:0:(namei.c:1399:ll_atomic_open()) VFS Op:name=f14.sanity-pfl, dir=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), file ffff8f9960493980, open_flags a141, mode 81b6 opened 0 00000080:00200000:1.0:1775764880.678505:0:24022:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=f14.sanity-pfl, dir=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), intent=open|creat 00000002:00010000:1.0:1775764880.678524:0:24022:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f14.sanity-pfl,[0x200000401:0xee3:0x0]) in obj [0x200000401:0xee2:0x0], intent: open|creat flags 0120502 00010000:00010000:1.0:1775764880.678535:0:24022:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d400/0x36f9bdc9e421ca71 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xee2:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fc6a3 expref: -99 pid: 24022 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764880.678548:0:24022:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea600/0x36f9bdc9e421ca2b lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xee2:0x0].0x0 bits 0x11/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc53e expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764880.678585:0:24022:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d400/0x36f9bdc9e421ca71 lrc: 1/0,0 mode: --/PR res: [0x200000401:0xee2:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fc6a3 expref: -99 pid: 24022 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764880.678589:0:24022:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764880.678591:0:24022:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d400/0x36f9bdc9e421ca71 lrc: 0/0,0 mode: --/PR res: [0x200000401:0xee2:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x2aac6a50e5fc6a3 expref: -99 pid: 24022 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764880.678630:0:24022:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d000/0x36f9bdc9e421ca78 lrc: 3/0,1 mode: --/CW res: [0x200000401:0xee2:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.678636:0:24022:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d000/0x36f9bdc9e421ca78 lrc: 3/0,1 mode: --/CW res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764880.678646:0:24022:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764880.678676:0:24022:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e6680 x1862022967063296/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 528/66744 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764880.678685:0:24022:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d000/0x36f9bdc9e421ca78 lrc: 3/0,1 mode: --/CW res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764880.678705:0:24022:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24022:1862022967063296:192.168.202.146@tcp:101:lfs.0 00000100:00100000:1.0:1775764880.679130:0:24022:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560240 going to sleep for 11 seconds 00000100:00100000:1.0:1775764880.684164:0:24022:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24022:1862022967063296:192.168.202.146@tcp:101:lfs.0 00010000:00010000:1.0:1775764880.684190:0:24022:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d000/0x36f9bdc9e421ca78 lrc: 4/0,1 mode: --/CW res: [0x200000401:0xee2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc6bf expref: -99 pid: 24022 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.684212:0:24022:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d000/0x36f9bdc9e421ca78 lrc: 4/0,1 mode: --/PR res: [0x200000401:0xee3:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc6bf expref: -99 pid: 24022 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.684221:0:24022:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d000/0x36f9bdc9e421ca78 lrc: 4/0,1 mode: --/PR res: [0x200000401:0xee3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc6bf expref: -99 pid: 24022 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.684234:0:24022:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d000/0x36f9bdc9e421ca78 lrc: 4/0,1 mode: PR/PR res: [0x200000401:0xee3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc6bf expref: -99 pid: 24022 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.684246:0:24022:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d000/0x36f9bdc9e421ca78 lrc: 4/0,1 mode: PR/PR res: [0x200000401:0xee3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc6bf expref: -99 pid: 24022 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.684261:0:24022:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d000/0x36f9bdc9e421ca78 lrc: 5/1,1 mode: PR/PR res: [0x200000401:0xee3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc6bf expref: -99 pid: 24022 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.684270:0:24022:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d000/0x36f9bdc9e421ca78 lrc: 5/1,1 mode: PR/PR res: [0x200000401:0xee3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc6bf expref: -99 pid: 24022 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.684316:0:24022:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d000/0x36f9bdc9e421ca78 lrc: 4/1,0 mode: PR/PR res: [0x200000401:0xee3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc6bf expref: -99 pid: 24022 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764880.684327:0:24022:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=3 disposition=8000037, status=0 req@ffff8f994f2e6680 x1862022967063296/t8589934636(8589934636) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 528/608 e 0 to 0 dl 1775764896 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764880.684338:0:24022:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f994f2e6680 x1862022967063296/t8589934636(8589934636) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 528/608 e 0 to 0 dl 1775764896 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764880.684350:0:24022:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d000/0x36f9bdc9e421ca78 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xee3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc6bf expref: -99 pid: 24022 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.684359:0:24022:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 4 res 8589935617/3811 (0 0) 00000080:00200000:1.0:1775764880.684366:0:24022:0:(namei.c:943:ll_intent_lock()) intent lock 3 on i1 [0x200000401:0xee2:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1775764880.684392:0:24022:0:(namei.c:1610:ll_create_it()) VFS Op:name=f14.sanity-pfl, dir=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), intent=open|creat 00000080:00200000:1.0:1775764880.684423:0:24022:0:(llite_lib.c:3177:ll_read_inode2()) VFS Op:inode=[0x200000401:0xee3:0x0](ffff8f9952e39988) 00000080:00200000:1.0:1775764880.684430:0:24022:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0xee3:0x0], updating i_size 0 00000080:00200000:1.0:1775764880.684523:0:24022:0:(file.c:6766:ll_layout_conf()) [0x200000401:0xee3:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:1.0:1775764880.684534:0:24022:0:(namei.c:1576:ll_create_node()) setting l_ast_data to inode [0x200000401:0xee3:0x0](ffff8f9952e39988) 00000080:00010000:1.0:1775764880.684537:0:24022:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xee3:0x0](ffff8f9952e39988) for lock 0x36f9bdc9e421ca78 00000080:00010000:1.0:1775764880.684548:0:24022:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000401:0xee3:0x0](ffff8f9952e39988) 00000080:00200008:1.0:1775764880.684566:0:24022:0:(file.c:1057:ll_file_open()) START file f14.sanity-pfl:[0x200000401:0xee3:0x0](ffff8f9952e39988), flags 120501 00000002:00100000:1.0:1775764880.684585:0:24022:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f994f2e6680 x1862022967063296/t8589934636(8589934636) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 528/608 e 0 to 0 dl 1775764896 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1775764880.684647:0:24022:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421ca78 from it ffff8f994ed57b40 00010000:00010000:1.0:1775764880.684652:0:24022:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d000/0x36f9bdc9e421ca78 lrc: 3/1,0 mode: PR/PR res: [0x200000401:0xee3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x2aac6a50e5fc6bf expref: -99 pid: 24022 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.684940:0:24022:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d000/0x36f9bdc9e421ca78 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xee3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x2aac6a50e5fc6bf expref: -99 pid: 24022 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764880.684961:0:24022:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xee3:0x0](ffff8f9952e39988), inode mode 81a4 mask 40 00000080:00200000:1.0:1775764880.685024:0:24022:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xee3:0x0](ffff8f9952e39988), inode mode 81a4 mask 2 00000080:00200000:1.0:1775764880.685034:0:24022:0:(xattr.c:410:ll_xattr_set()) VFS Op:inode=[0x200000401:0xee3:0x0](ffff8f9952e39988), xattr lov 00000002:00010000:1.0:1775764880.685049:0:24022:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xee3:0x0]) in obj [0x200000401:0xee2:0x0], intent: open flags 040000002 00010000:00010000:1.0:1775764880.685091:0:24022:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695cc00/0x36f9bdc9e421ca7f lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.685099:0:24022:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695cc00/0x36f9bdc9e421ca7f lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764880.685109:0:24022:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764880.685115:0:24022:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e7100 x1862022967063552/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1072/132280 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764880.685125:0:24022:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695cc00/0x36f9bdc9e421ca7f lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764880.685146:0:24022:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7100 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24022:1862022967063552:192.168.202.146@tcp:101:lfs.0 00000100:00100000:1.0:1775764880.685880:0:24022:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560240 going to sleep for 11 seconds 00000100:00100000:0.0:1775764880.688000:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764880.688051:0:5562:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978434432 00000100:00100000:1.0:1775764880.688060:0:5562:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978434432 00000100:00100000:1.0:1775764880.688120:0:5562:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24233 00000100:00100000:1.0:1775764880.688131:0:5562:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f9970a77b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+5:17504:x1862022978434432:12345-192.168.202.146@tcp:104: 00010000:00010000:1.0:1775764880.688147:0:5562:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d000/0x36f9bdc9e421ca78 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xee3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x2aac6a50e5fc6bf expref: -99 pid: 24022 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764880.688829:0:5562:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764880.688838:0:5562:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f9970a77b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:3f83546c-01a9-4156-8a97-44c446245d3e+5:17504:x1862022978434432:12345-192.168.202.146@tcp:104: Request processed in 707us (843us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764880.688845:0:5562:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24233 00010000:00010000:3.0:1775764880.688909:0:19801:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1775764880.688918:0:19801:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d000/0x36f9bdc9e421ca78 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xee3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x2aac6a50e5fc6bf expref: -99 pid: 24022 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764880.688927:0:19801:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f996695d000 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:3.0:1775764880.688933:0:19801:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d000/0x36f9bdc9e421ca78 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xee3:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x2aac6a50e5fc6bf expref: -99 pid: 24022 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764880.688963:0:19801:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d000/0x36f9bdc9e421ca78 lrc: 2/0,0 mode: PR/PR res: [0x200000401:0xee3:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x2aac6a50e5fc6bf expref: -99 pid: 24022 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764880.689051:0:19801:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d000/0x36f9bdc9e421ca78 lrc: 3/0,0 mode: PR/PR res: [0x200000401:0xee3:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x2aac6a50e5fc6bf expref: -99 pid: 24022 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1775764880.689100:0:19801:0:(vvp_object.c:117:vvp_conf_set()) [0x200000401:0xee3:0x0]: losing layout lock 00010000:00010000:3.0:1775764880.689156:0:19801:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d000/0x36f9bdc9e421ca78 lrc: 2/0,0 mode: --/PR res: [0x200000401:0xee3:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x2aac6a50e5fc6bf expref: -99 pid: 24022 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764880.689164:0:19801:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764880.689201:0:19801:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d000/0x36f9bdc9e421ca78 lrc: 1/0,0 mode: --/PR res: [0x200000401:0xee3:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x2aac6a50e5fc6bf expref: -99 pid: 24022 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764880.689208:0:19801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d000/0x36f9bdc9e421ca78 lrc: 0/0,0 mode: --/PR res: [0x200000401:0xee3:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x2aac6a50e5fc6bf expref: -99 pid: 24022 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1775764880.689229:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764880.689231:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764880.689233:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764880.689243:0:2382:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1775764880.689259:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967063680:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764880.692883:0:24022:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7100 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24022:1862022967063552:192.168.202.146@tcp:101:lfs.0 00010000:00010000:2.0:1775764880.692907:0:24022:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695cc00/0x36f9bdc9e421ca7f lrc: 4/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764880.692919:0:24022:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695cc00/0x36f9bdc9e421ca7f lrc: 4/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764880.692926:0:24022:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695cc00/0x36f9bdc9e421ca7f lrc: 4/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764880.692933:0:24022:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695cc00/0x36f9bdc9e421ca7f lrc: 3/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764880.692972:0:24022:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695cc00/0x36f9bdc9e421ca7f lrc: 4/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764880.692979:0:24022:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f996695cc00 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:2.0:1775764880.692984:0:24022:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695cc00/0x36f9bdc9e421ca7f lrc: 4/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764880.692991:0:24022:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695cc00/0x36f9bdc9e421ca7f lrc: 4/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764880.692998:0:24022:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695cc00/0x36f9bdc9e421ca7f lrc: 5/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764880.693006:0:24022:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1775764880.693013:0:24022:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695cc00/0x36f9bdc9e421ca7f lrc: 3/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764880.693019:0:24022:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695cc00/0x36f9bdc9e421ca7f lrc: 0/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24022 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764880.693038:0:24022:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=800002b, status=0 req@ffff8f994f2e7100 x1862022967063552/t8589934637(8589934637) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1072/1120 e 0 to 0 dl 1775764896 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764880.693053:0:24022:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f994f2e7100 x1862022967063552/t8589934637(8589934637) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1072/1120 e 0 to 0 dl 1775764896 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764880.693071:0:24022:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0xee2:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764880.693187:0:24022:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xee3:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764880.693196:0:24022:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xee3:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764880.693264:0:24022:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0xee3:0x0], updating i_size 0 00000002:00100000:2.0:1775764880.693294:0:24022:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f994f2e7100 x1862022967063552/t8589934637(8589934637) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1032/1120 e 0 to 0 dl 1775764896 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764880.693332:0:24022:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f994f2e7100 x1862022967063552/t8589934637(8589934637) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1032/1120 e 0 to 0 dl 1775764896 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764880.693342:0:24022:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764880.693346:0:24022:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f99703d2680 x1862022967063936/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/1048 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764880.693371:0:24022:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24022:1862022967063936:192.168.202.146@tcp:35:lfs.0 00000100:00100000:2.0:1775764880.693724:0:24022:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9966b46240 going to sleep for 11 seconds 00000100:00100000:3.0:1775764880.694593:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967063680:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764880.696705:0:24022:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24022:1862022967063936:192.168.202.146@tcp:35:lfs.0 00000002:00100000:2.0:1775764880.696736:0:24022:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f994f2e7100 x1862022967063552/t8589934637(8589934637) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1032/1120 e 0 to 0 dl 1775764896 ref 2 fl Complete:RQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200008:2.0:1775764880.696819:0:24022:0:(file.c:447:ll_file_release()) START file f14.sanity-pfl:[0x200000401:0xee3:0x0](ffff8f9952e39988), flags 100001 00000002:00100000:2.0:1775764880.696871:0:24022:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f994f2e6680 x1862022967063296/t8589934636(8589934636) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 528/608 e 0 to 0 dl 1775764896 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764880.696880:0:24022:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764880.696885:0:24022:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f99703d0380 x1862022967064192/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/1048 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764880.696904:0:24022:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24022:1862022967064192:192.168.202.146@tcp:35:lfs.0 00000100:00100000:2.0:1775764880.696991:0:24022:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9966b46240 going to sleep for 11 seconds 00000100:00100000:2.0:1775764880.698500:0:24022:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24022:1862022967064192:192.168.202.146@tcp:35:lfs.0 00000002:00100000:2.0:1775764880.698525:0:24022:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f994f2e6680 x1862022967063296/t8589934636(8589934636) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 528/608 e 0 to 0 dl 1775764896 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764880.705563:0:24023:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764880.705595:0:24023:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764880.705625:0:24023:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764880.705641:0:24023:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1775764880.705650:0:24023:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764880.705659:0:24023:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764880.705662:0:24023:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421ca24 00000080:00010000:1.0:1775764880.705669:0:24023:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421ca24 from it ffffafbac1c63bf8 00010000:00010000:1.0:1775764880.705673:0:24023:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764880.705682:0:24023:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:1.0:1775764880.705697:0:24023:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764880.705704:0:24023:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d14.sanity-pfl, flags=27 00000080:00200000:1.0:1775764880.705710:0:24023:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764880.705722:0:24023:0:(namei.c:1399:ll_atomic_open()) VFS Op:name=f14.sanity-pfl, dir=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), file ffff8f99439ca180, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1775764880.705779:0:24023:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=f14.sanity-pfl, dir=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), intent=open 00000002:00010000:1.0:1775764880.705793:0:24023:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f14.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000401:0xee2:0x0], intent: open flags 0304001 00010000:00010000:1.0:1775764880.705845:0:24023:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d600/0x36f9bdc9e421ca86 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24023 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.705853:0:24023:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d600/0x36f9bdc9e421ca86 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24023 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764880.705865:0:24023:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764880.705879:0:24023:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e4a80 x1862022967064448/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1080/132280 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764880.705889:0:24023:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d600/0x36f9bdc9e421ca86 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24023 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764880.705911:0:24023:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24023:1862022967064448:192.168.202.146@tcp:101:lfs.0 00000100:00100000:1.0:1775764880.706549:0:24023:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560b40 going to sleep for 11 seconds 00000100:00100000:1.0:1775764880.709707:0:24023:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24023:1862022967064448:192.168.202.146@tcp:101:lfs.0 00010000:00010000:1.0:1775764880.709728:0:24023:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d600/0x36f9bdc9e421ca86 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24023 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.709798:0:24023:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d600/0x36f9bdc9e421ca86 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24023 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.709804:0:24023:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d600/0x36f9bdc9e421ca86 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24023 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.709809:0:24023:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d600/0x36f9bdc9e421ca86 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24023 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.709851:0:24023:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d600/0x36f9bdc9e421ca86 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24023 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.709856:0:24023:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f996695d600 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:1.0:1775764880.709859:0:24023:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d600/0x36f9bdc9e421ca86 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 24023 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.709864:0:24023:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d600/0x36f9bdc9e421ca86 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 24023 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.709869:0:24023:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d600/0x36f9bdc9e421ca86 lrc: 5/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24023 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.709873:0:24023:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1775764880.709877:0:24023:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d600/0x36f9bdc9e421ca86 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24023 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.709881:0:24023:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d600/0x36f9bdc9e421ca86 lrc: 0/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24023 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764880.709894:0:24023:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff8f994f2e4a80 x1862022967064448/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1080/608 e 0 to 0 dl 1775764896 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764880.709905:0:24023:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0xee2:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1775764880.710038:0:24023:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764880.710063:0:24023:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764880.710076:0:24023:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:1.0:1775764880.710085:0:24023:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:1.0:1775764880.710100:0:24023:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764880.710105:0:24023:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764880.710108:0:24023:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421ca24 00000080:00010000:1.0:1775764880.710112:0:24023:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421ca24 from it ffffafbac1c63bf8 00010000:00010000:1.0:1775764880.710114:0:24023:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:1.0:1775764880.710120:0:24023:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:1.0:1775764880.710130:0:24023:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764880.710135:0:24023:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d14.sanity-pfl, flags=267 00000080:00200000:1.0:1775764880.710140:0:24023:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), inode mode 41ed mask 44 00000080:00200008:1.0:1775764880.710145:0:24023:0:(file.c:1057:ll_file_open()) START file d14.sanity-pfl:[0x200000401:0xee2:0x0](ffff8f9952e3ba88), flags 304000 00000002:00010000:1.0:1775764880.710156:0:24023:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xee2:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100240304001 00010000:00010000:1.0:1775764880.710188:0:24023:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e421ca8d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24023 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.710193:0:24023:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e421ca8d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24023 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764880.710201:0:24023:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764880.710205:0:24023:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e4a80 x1862022967064704/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1072/132280 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764880.710212:0:24023:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e421ca8d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24023 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764880.710230:0:24023:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24023:1862022967064704:192.168.202.146@tcp:101:lfs.0 00000100:00100000:1.0:1775764880.710367:0:24023:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560b40 going to sleep for 11 seconds 00000100:00100000:1.0:1775764880.712535:0:24023:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24023:1862022967064704:192.168.202.146@tcp:101:lfs.0 00010000:00010000:1.0:1775764880.712550:0:24023:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e421ca8d lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24023 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.712559:0:24023:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e421ca8d lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24023 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.712563:0:24023:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e421ca8d lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24023 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.712568:0:24023:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e421ca8d lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24023 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.712573:0:24023:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e421ca8d lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24023 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.712577:0:24023:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f996695c200 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:1.0:1775764880.712580:0:24023:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e421ca8d lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 24023 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.712583:0:24023:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e421ca8d lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 24023 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.712588:0:24023:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e421ca8d lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24023 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.712592:0:24023:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1775764880.712636:0:24023:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e421ca8d lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24023 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.712641:0:24023:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e421ca8d lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24023 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764880.712653:0:24023:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8f994f2e4a80 x1862022967064704/t8589934640(8589934640) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1072/608 e 0 to 0 dl 1775764896 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764880.712662:0:24023:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f994f2e4a80 x1862022967064704/t8589934640(8589934640) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1072/608 e 0 to 0 dl 1775764896 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764880.712671:0:24023:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1775764880.712689:0:24023:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0xee2:0x0], updating i_size 7680 00000002:00100000:1.0:1775764880.712702:0:24023:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f994f2e4a80 x1862022967064704/t8589934640(8589934640) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1072/608 e 0 to 0 dl 1775764896 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:1.0:1775764880.713078:0:24023:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000401:0xee2:0x0](ffff8f9952e3ba88) cmd=c0086915 arg=1cfa3d0 00000100:00100000:1.0:1775764880.713129:0:24023:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5880 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24023:1862022967064832:192.168.202.146@tcp:34:lfs.0 00000100:00100000:1.0:1775764880.713184:0:24023:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560b40 going to sleep for 11 seconds 00000100:00100000:1.0:1775764880.715653:0:24023:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5880 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24023:1862022967064832:192.168.202.146@tcp:34:lfs.0 00000080:00200008:1.0:1775764880.715881:0:24023:0:(file.c:447:ll_file_release()) START file d14.sanity-pfl:[0x200000401:0xee2:0x0](ffff8f9952e3ba88), flags 304000 00000002:00100000:1.0:1775764880.715946:0:24023:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f994f2e4a80 x1862022967064704/t8589934640(8589934640) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1072/608 e 0 to 0 dl 1775764896 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1775764880.715959:0:24023:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764880.715964:0:24023:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e5880 x1862022967065088/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/1048 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764880.715985:0:24023:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5880 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24023:1862022967065088:192.168.202.146@tcp:35:lfs.0 00000100:00100000:1.0:1775764880.716112:0:24023:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560b40 going to sleep for 11 seconds 00000100:00100000:1.0:1775764880.718309:0:24023:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5880 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24023:1862022967065088:192.168.202.146@tcp:35:lfs.0 00000002:00100000:1.0:1775764880.718335:0:24023:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8f994f2e4a80 x1862022967064704/t8589934640(8589934640) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1072/608 e 0 to 0 dl 1775764896 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764880.725232:0:24024:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764880.725256:0:24024:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764880.725273:0:24024:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764880.725287:0:24024:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:1.0:1775764880.725292:0:24024:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764880.725300:0:24024:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764880.725303:0:24024:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421ca24 00000080:00010000:1.0:1775764880.725309:0:24024:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421ca24 from it ffffafbac1cabbf8 00010000:00010000:1.0:1775764880.725312:0:24024:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764880.725319:0:24024:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00000080:00200000:1.0:1775764880.725330:0:24024:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764880.725335:0:24024:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d14.sanity-pfl, flags=27 00000080:00200000:1.0:1775764880.725339:0:24024:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764880.725347:0:24024:0:(namei.c:1399:ll_atomic_open()) VFS Op:name=f14.sanity-pfl, dir=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), file ffff8f9943604900, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1775764880.725388:0:24024:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=f14.sanity-pfl, dir=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), intent=open 00000002:00010000:1.0:1775764880.725399:0:24024:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f14.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000401:0xee2:0x0], intent: open flags 0304001 00010000:00010000:1.0:1775764880.725443:0:24024:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e421ca94 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24024 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.725451:0:24024:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e421ca94 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24024 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764880.725462:0:24024:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764880.725474:0:24024:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e5880 x1862022967065344/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1080/132280 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764880.725485:0:24024:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e421ca94 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24024 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764880.725507:0:24024:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5880 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24024:1862022967065344:192.168.202.146@tcp:101:lfs.0 00000100:00100000:1.0:1775764880.725725:0:24024:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560b40 going to sleep for 11 seconds 00000100:00100000:1.0:1775764880.733500:0:24024:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5880 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24024:1862022967065344:192.168.202.146@tcp:101:lfs.0 00010000:00010000:1.0:1775764880.733523:0:24024:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e421ca94 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24024 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.733533:0:24024:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e421ca94 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24024 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.733538:0:24024:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e421ca94 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24024 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.733543:0:24024:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e421ca94 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 24024 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.733567:0:24024:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e421ca94 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 24024 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.733572:0:24024:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f996695c200 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:1.0:1775764880.733576:0:24024:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e421ca94 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 24024 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.733581:0:24024:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e421ca94 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 24024 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.733585:0:24024:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e421ca94 lrc: 5/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 24024 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.733590:0:24024:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1775764880.733595:0:24024:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e421ca94 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24024 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.733598:0:24024:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e421ca94 lrc: 0/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 24024 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764880.733623:0:24024:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff8f994f2e5880 x1862022967065344/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1080/608 e 0 to 0 dl 1775764896 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764880.733635:0:24024:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0xee2:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1775764880.733802:0:24024:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764880.733820:0:24024:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764880.733834:0:24024:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00010000:00010000:1.0:1775764880.733843:0:24024:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00000080:00200000:1.0:1775764880.733861:0:24024:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764880.733866:0:24024:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764880.733869:0:24024:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421ca24 00000080:00010000:1.0:1775764880.733872:0:24024:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421ca24 from it ffffafbac1cabbf8 00010000:00010000:1.0:1775764880.733874:0:24024:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00010000:00010000:1.0:1775764880.733880:0:24024:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 0 00000080:00200000:1.0:1775764880.733891:0:24024:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764880.733896:0:24024:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d14.sanity-pfl, flags=267 00000080:00200000:1.0:1775764880.733902:0:24024:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), inode mode 41ed mask 44 00000080:00200008:1.0:1775764880.733908:0:24024:0:(file.c:1057:ll_file_open()) START file d14.sanity-pfl:[0x200000401:0xee2:0x0](ffff8f9952e3ba88), flags 304000 00000002:00010000:1.0:1775764880.733920:0:24024:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000401:0xee2:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0104240304001 00010000:00010000:1.0:1775764880.733961:0:24024:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d600/0x36f9bdc9e421ca9b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 24024 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.733969:0:24024:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d600/0x36f9bdc9e421ca9b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24024 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764880.733980:0:24024:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764880.733986:0:24024:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e5880 x1862022967065600/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1072/132280 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764880.733995:0:24024:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d600/0x36f9bdc9e421ca9b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 24024 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764880.734026:0:24024:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5880 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24024:1862022967065600:192.168.202.146@tcp:101:lfs.0 00000100:00100000:1.0:1775764880.734174:0:24024:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560b40 going to sleep for 11 seconds 00000100:00100000:1.0:1775764880.738927:0:24024:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5880 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24024:1862022967065600:192.168.202.146@tcp:101:lfs.0 00010000:00010000:1.0:1775764880.738962:0:24024:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d600/0x36f9bdc9e421ca9b lrc: 4/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc736 expref: -99 pid: 24024 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.738974:0:24024:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d600/0x36f9bdc9e421ca9b lrc: 4/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc736 expref: -99 pid: 24024 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.738986:0:24024:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d600/0x36f9bdc9e421ca9b lrc: 4/1,0 mode: CR/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc736 expref: -99 pid: 24024 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.738995:0:24024:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d600/0x36f9bdc9e421ca9b lrc: 4/1,0 mode: CR/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc736 expref: -99 pid: 24024 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764880.739006:0:24024:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=200002b, status=0 req@ffff8f994f2e5880 x1862022967065600/t8589934642(8589934642) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1072/608 e 0 to 0 dl 1775764896 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764880.739015:0:24024:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f994f2e5880 x1862022967065600/t8589934642(8589934642) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1072/608 e 0 to 0 dl 1775764896 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764880.739023:0:24024:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d600/0x36f9bdc9e421ca9b lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc736 expref: -99 pid: 24024 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.739029:0:24024:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8f995870a800 type 13 mode 16 res 8589935617/3810 (0 0) 00000080:00200000:1.0:1775764880.739034:0:24024:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1775764880.739060:0:24024:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000401:0xee2:0x0], updating i_size 7680 00000080:00010000:1.0:1775764880.739099:0:24024:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000401:0xee2:0x0](ffff8f9952e3ba88) for lock 0x36f9bdc9e421ca9b 00000080:00010000:1.0:1775764880.739105:0:24024:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421ca9b from it ffffafbac1cabbd0 00010000:00010000:1.0:1775764880.739108:0:24024:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d600/0x36f9bdc9e421ca9b lrc: 3/1,0 mode: CR/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc736 expref: -99 pid: 24024 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.739114:0:24024:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d600/0x36f9bdc9e421ca9b lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc736 expref: -99 pid: 24024 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000002:00100000:1.0:1775764880.739134:0:24024:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8f994f2e5880 x1862022967065600/t8589934642(8589934642) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1072/608 e 0 to 0 dl 1775764896 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:1.0:1775764880.739214:0:24024:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000401:0xee2:0x0](ffff8f9952e3ba88) cmd=c0086915 arg=7cf3d0 00000100:00100000:1.0:1775764880.739264:0:24024:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24024:1862022967065728:192.168.202.146@tcp:34:lfs.0 00000100:00100000:1.0:1775764880.739329:0:24024:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560b40 going to sleep for 11 seconds 00000100:00100000:1.0:1775764880.742335:0:24024:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24024:1862022967065728:192.168.202.146@tcp:34:lfs.0 00000080:00200008:1.0:1775764880.742672:0:24024:0:(file.c:447:ll_file_release()) START file d14.sanity-pfl:[0x200000401:0xee2:0x0](ffff8f9952e3ba88), flags 304000 00010000:00010000:1.0:1775764880.742704:0:24024:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d600/0x36f9bdc9e421ca9b lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc736 expref: -99 pid: 24024 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1775764880.749704:0:24025:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:3.0:1775764880.749730:0:24025:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764880.749808:0:24025:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 0 00010000:00010000:3.0:1775764880.749863:0:24025:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 0 00000080:00200000:3.0:1775764880.749869:0:24025:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764880.749878:0:24025:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:3.0:1775764880.749880:0:24025:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421ca24 00000080:00010000:3.0:1775764880.749887:0:24025:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421ca24 from it ffffafbac1c73bf8 00010000:00010000:3.0:1775764880.749889:0:24025:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 0 00010000:00010000:3.0:1775764880.749896:0:24025:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:3.0:1775764880.749907:0:24025:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:3.0:1775764880.749913:0:24025:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d14.sanity-pfl, flags=27 00000080:00200000:3.0:1775764880.749917:0:24025:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764880.749927:0:24025:0:(namei.c:1399:ll_atomic_open()) VFS Op:name=f14.sanity-pfl, dir=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), file ffff8f99613aaa80, open_flags 18800, mode 0 opened 0 00000080:00200000:3.0:1775764880.749934:0:24025:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=f14.sanity-pfl, dir=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), intent=open 00000002:00010000:3.0:1775764880.749948:0:24025:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f14.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000401:0xee2:0x0], intent: open flags 0304001 00010000:00010000:3.0:1775764880.750003:0:24025:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421caa2 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 24025 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764880.750009:0:24025:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421caa2 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 24025 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:3.0:1775764880.750018:0:24025:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1775764880.750028:0:24025:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970edc000 x1862022967065984/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1080/132280 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1775764880.750049:0:24025:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421caa2 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 24025 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764880.750078:0:24025:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24025:1862022967065984:192.168.202.146@tcp:101:lfs.0 00000100:00100000:3.0:1775764880.750256:0:24025:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9945c06cc0 going to sleep for 11 seconds 00000100:00100000:3.0:1775764880.754818:0:24025:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24025:1862022967065984:192.168.202.146@tcp:101:lfs.0 00010000:00010000:3.0:1775764880.754843:0:24025:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421caa2 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 24025 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764880.754853:0:24025:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421caa2 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 24025 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764880.754859:0:24025:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421caa2 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 24025 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764880.754863:0:24025:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421caa2 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 24025 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764880.754869:0:24025:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421caa2 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 24025 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764880.754872:0:24025:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99435d4e00 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:3.0:1775764880.754876:0:24025:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421caa2 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 24025 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764880.754882:0:24025:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421caa2 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 24025 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764880.754887:0:24025:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421caa2 lrc: 5/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 24025 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764880.754922:0:24025:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1775764880.754928:0:24025:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421caa2 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 24025 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764880.754934:0:24025:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4e00/0x36f9bdc9e421caa2 lrc: 0/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 24025 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764880.754953:0:24025:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff8f9970edc000 x1862022967065984/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1080/608 e 0 to 0 dl 1775764896 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1775764880.754968:0:24025:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0xee2:0x0] suppgids 0 -1: rc -20 00000080:00200000:3.0:1775764880.755103:0:24025:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:3.0:1775764880.755122:0:24025:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764880.755134:0:24025:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:3.0:1775764880.755143:0:24025:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:3.0:1775764880.755150:0:24025:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764880.755156:0:24025:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:3.0:1775764880.755159:0:24025:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421ca24 00000080:00010000:3.0:1775764880.755163:0:24025:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421ca24 from it ffffafbac1c73bf8 00010000:00010000:3.0:1775764880.755166:0:24025:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:3.0:1775764880.755174:0:24025:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:3.0:1775764880.755206:0:24025:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:3.0:1775764880.755213:0:24025:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d14.sanity-pfl, flags=267 00000080:00200000:3.0:1775764880.755219:0:24025:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), inode mode 41ed mask 44 00000080:00200008:3.0:1775764880.755227:0:24025:0:(file.c:1057:ll_file_open()) START file d14.sanity-pfl:[0x200000401:0xee2:0x0](ffff8f9952e3ba88), flags 304000 00000080:00200080:3.0:1775764880.755266:0:24025:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000401:0xee2:0x0](ffff8f9952e3ba88) cmd=c0086915 arg=24b03d0 00000100:00100000:3.0:1775764880.755320:0:24025:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24025:1862022967066112:192.168.202.146@tcp:34:lfs.0 00000100:00100000:3.0:1775764880.755421:0:24025:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9945c06cc0 going to sleep for 11 seconds 00000100:00100000:3.0:1775764880.758577:0:24025:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24025:1862022967066112:192.168.202.146@tcp:34:lfs.0 00000080:00200008:3.0:1775764880.758969:0:24025:0:(file.c:447:ll_file_release()) START file d14.sanity-pfl:[0x200000401:0xee2:0x0](ffff8f9952e3ba88), flags 304000 00010000:00010000:3.0:1775764880.759000:0:24025:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d600/0x36f9bdc9e421ca9b lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc736 expref: -99 pid: 24024 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1775764880.767364:0:24026:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:3.0:1775764880.767489:0:24026:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1775764880.767508:0:24026:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:3.0:1775764880.767522:0:24026:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:3.0:1775764880.767527:0:24026:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1775764880.767533:0:24026:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:3.0:1775764880.767536:0:24026:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421ca24 00000080:00010000:3.0:1775764880.767541:0:24026:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421ca24 from it ffffafbac1c3bbf8 00010000:00010000:3.0:1775764880.767544:0:24026:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:3.0:1775764880.767588:0:24026:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00200000:3.0:1775764880.767609:0:24026:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:3.0:1775764880.767616:0:24026:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d14.sanity-pfl, flags=27 00000080:00200000:3.0:1775764880.767621:0:24026:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), inode mode 41ed mask 1 00000080:00200000:3.0:1775764880.767630:0:24026:0:(namei.c:1399:ll_atomic_open()) VFS Op:name=f14.sanity-pfl, dir=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), file ffff8f9952bc8480, open_flags 18800, mode 0 opened 0 00000080:00200000:3.0:1775764880.767635:0:24026:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=f14.sanity-pfl, dir=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), intent=open 00000002:00010000:3.0:1775764880.767648:0:24026:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f14.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000401:0xee2:0x0], intent: open flags 0304001 00010000:00010000:3.0:1775764880.767703:0:24026:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421caa9 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 24026 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764880.767709:0:24026:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421caa9 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 24026 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:3.0:1775764880.767719:0:24026:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1775764880.767729:0:24026:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f9970edc000 x1862022967066368/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1080/132280 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1775764880.767735:0:24026:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421caa9 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 24026 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764880.767790:0:24026:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24026:1862022967066368:192.168.202.146@tcp:101:lfs.0 00000100:00100000:3.0:1775764880.767949:0:24026:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9945c06cc0 going to sleep for 11 seconds 00000100:00100000:3.0:1775764880.771704:0:24026:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24026:1862022967066368:192.168.202.146@tcp:101:lfs.0 00010000:00010000:3.0:1775764880.771727:0:24026:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421caa9 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 24026 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764880.771834:0:24026:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421caa9 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 24026 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764880.771841:0:24026:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421caa9 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 24026 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764880.771848:0:24026:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421caa9 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 24026 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764880.771858:0:24026:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421caa9 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 24026 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764880.771864:0:24026:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99435d4800 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:3.0:1775764880.771868:0:24026:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421caa9 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 24026 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764880.771875:0:24026:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421caa9 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 24026 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764880.771881:0:24026:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421caa9 lrc: 5/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 24026 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764880.771885:0:24026:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1775764880.771890:0:24026:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421caa9 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 24026 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764880.771894:0:24026:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99435d4800/0x36f9bdc9e421caa9 lrc: 0/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 24026 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1775764880.771907:0:24026:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff8f9970edc000 x1862022967066368/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1080/608 e 0 to 0 dl 1775764896 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1775764880.771986:0:24026:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0xee2:0x0] suppgids 0 -1: rc -20 00000080:00200000:2.0:1775764880.772194:0:24026:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:2.0:1775764880.772211:0:24026:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764880.772225:0:24026:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:2.0:1775764880.772234:0:24026:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00200000:2.0:1775764880.772239:0:24026:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764880.772243:0:24026:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:2.0:1775764880.772245:0:24026:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421ca24 00000080:00010000:2.0:1775764880.772249:0:24026:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421ca24 from it ffffafbac1c3bbf8 00010000:00010000:2.0:1775764880.772252:0:24026:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:2.0:1775764880.772257:0:24026:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00000080:00200000:2.0:1775764880.772266:0:24026:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:2.0:1775764880.772273:0:24026:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d14.sanity-pfl, flags=267 00000080:00200000:2.0:1775764880.772280:0:24026:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), inode mode 41ed mask 44 00000080:00200008:2.0:1775764880.772287:0:24026:0:(file.c:1057:ll_file_open()) START file d14.sanity-pfl:[0x200000401:0xee2:0x0](ffff8f9952e3ba88), flags 304000 00000080:00200080:2.0:1775764880.772325:0:24026:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000401:0xee2:0x0](ffff8f9952e3ba88) cmd=c0086915 arg=24e03d0 00000100:00100000:2.0:1775764880.772456:0:24026:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24026:1862022967066496:192.168.202.146@tcp:34:lfs.0 00000100:00100000:2.0:1775764880.772563:0:24026:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9966b46240 going to sleep for 11 seconds 00000100:00100000:2.0:1775764880.775635:0:24026:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24026:1862022967066496:192.168.202.146@tcp:34:lfs.0 00000080:00200008:2.0:1775764880.775857:0:24026:0:(file.c:447:ll_file_release()) START file d14.sanity-pfl:[0x200000401:0xee2:0x0](ffff8f9952e3ba88), flags 304000 00010000:00010000:2.0:1775764880.775887:0:24026:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d600/0x36f9bdc9e421ca9b lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc736 expref: -99 pid: 24024 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764880.785154:0:24027:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764880.785191:0:24027:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764880.785214:0:24027:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00010000:00010000:1.0:1775764880.785230:0:24027:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00000080:00200000:1.0:1775764880.785236:0:24027:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764880.785284:0:24027:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764880.785288:0:24027:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421ca24 00000080:00010000:1.0:1775764880.785296:0:24027:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421ca24 from it ffffafbac1cbbbf8 00010000:00010000:1.0:1775764880.785300:0:24027:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00010000:00010000:1.0:1775764880.785310:0:24027:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00000080:00200000:1.0:1775764880.785328:0:24027:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764880.785336:0:24027:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d14.sanity-pfl, flags=27 00000080:00200000:1.0:1775764880.785343:0:24027:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764880.785354:0:24027:0:(namei.c:1399:ll_atomic_open()) VFS Op:name=f14.sanity-pfl, dir=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), file ffff8f9959670600, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1775764880.785360:0:24027:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=f14.sanity-pfl, dir=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), intent=open 00000002:00010000:1.0:1775764880.785374:0:24027:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f14.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000401:0xee2:0x0], intent: open flags 0304001 00010000:00010000:1.0:1775764880.785435:0:24027:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e421cab0 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 24027 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.785444:0:24027:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e421cab0 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 24027 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764880.785458:0:24027:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764880.785470:0:24027:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f994f2e4a80 x1862022967066752/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1080/132280 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764880.785480:0:24027:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e421cab0 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 24027 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764880.785504:0:24027:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24027:1862022967066752:192.168.202.146@tcp:101:lfs.0 00000100:00100000:1.0:1775764880.785786:0:24027:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560b40 going to sleep for 11 seconds 00000100:00100000:1.0:1775764880.789077:0:24027:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24027:1862022967066752:192.168.202.146@tcp:101:lfs.0 00010000:00010000:1.0:1775764880.789101:0:24027:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e421cab0 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 24027 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.789113:0:24027:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e421cab0 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 24027 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.789118:0:24027:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e421cab0 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 24027 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.789123:0:24027:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e421cab0 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 24027 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.789128:0:24027:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e421cab0 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 24027 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.789132:0:24027:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f996695c200 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:1.0:1775764880.789135:0:24027:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e421cab0 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 24027 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.789140:0:24027:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e421cab0 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 24027 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.789147:0:24027:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e421cab0 lrc: 5/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 24027 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.789154:0:24027:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1775764880.789161:0:24027:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e421cab0 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 24027 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764880.789171:0:24027:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695c200/0x36f9bdc9e421cab0 lrc: 0/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 24027 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764880.789188:0:24027:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff8f994f2e4a80 x1862022967066752/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1080/608 e 0 to 0 dl 1775764896 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764880.789199:0:24027:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0xee2:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1775764880.789326:0:24027:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:1.0:1775764880.789343:0:24027:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764880.789356:0:24027:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00010000:00010000:1.0:1775764880.789365:0:24027:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00000080:00200000:1.0:1775764880.789369:0:24027:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764880.789374:0:24027:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:1.0:1775764880.789377:0:24027:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421ca24 00000080:00010000:1.0:1775764880.789382:0:24027:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421ca24 from it ffffafbac1cbbbf8 00010000:00010000:1.0:1775764880.789385:0:24027:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00010000:00010000:1.0:1775764880.789393:0:24027:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00000080:00200000:1.0:1775764880.789407:0:24027:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764880.789414:0:24027:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d14.sanity-pfl, flags=267 00000080:00200000:1.0:1775764880.789498:0:24027:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), inode mode 41ed mask 44 00000080:00200008:1.0:1775764880.789507:0:24027:0:(file.c:1057:ll_file_open()) START file d14.sanity-pfl:[0x200000401:0xee2:0x0](ffff8f9952e3ba88), flags 304000 00000080:00200080:1.0:1775764880.789540:0:24027:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000401:0xee2:0x0](ffff8f9952e3ba88) cmd=c0086915 arg=189c3d0 00000100:00100000:1.0:1775764880.789587:0:24027:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24027:1862022967066880:192.168.202.146@tcp:34:lfs.0 00000100:00100000:1.0:1775764880.789704:0:24027:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9943560b40 going to sleep for 11 seconds 00000100:00100000:1.0:1775764880.793154:0:24027:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24027:1862022967066880:192.168.202.146@tcp:34:lfs.0 00000080:00200008:1.0:1775764880.793325:0:24027:0:(file.c:447:ll_file_release()) START file d14.sanity-pfl:[0x200000401:0xee2:0x0](ffff8f9952e3ba88), flags 304000 00010000:00010000:1.0:1775764880.793353:0:24027:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d600/0x36f9bdc9e421ca9b lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc736 expref: -99 pid: 24024 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1775764880.806034:0:24028:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:2.0:1775764880.806064:0:24028:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764880.806083:0:24028:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00010000:00010000:2.0:1775764880.806099:0:24028:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00000080:00200000:2.0:1775764880.806107:0:24028:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764880.806115:0:24028:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:2.0:1775764880.806119:0:24028:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421ca24 00000080:00010000:2.0:1775764880.806125:0:24028:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421ca24 from it ffffafbac1ccbbf8 00010000:00010000:2.0:1775764880.806129:0:24028:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00010000:00010000:2.0:1775764880.806137:0:24028:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00000080:00200000:2.0:1775764880.806153:0:24028:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:2.0:1775764880.806161:0:24028:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d14.sanity-pfl, flags=27 00000080:00200000:2.0:1775764880.806167:0:24028:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), inode mode 41ed mask 1 00000080:00200000:2.0:1775764880.806180:0:24028:0:(namei.c:1399:ll_atomic_open()) VFS Op:name=f14.sanity-pfl, dir=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), file ffff8f994ee5c480, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1775764880.806186:0:24028:0:(namei.c:1063:ll_lookup_it()) VFS Op:name=f14.sanity-pfl, dir=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), intent=open 00000002:00010000:2.0:1775764880.806245:0:24028:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f14.sanity-pfl,[0x0:0x0:0x0]) in obj [0x200000401:0xee2:0x0], intent: open flags 0304001 00010000:00010000:2.0:1775764880.806504:0:24028:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebc00/0x36f9bdc9e421cab7 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 24028 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764880.806514:0:24028:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebc00/0x36f9bdc9e421cab7 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 24028 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764880.806527:0:24028:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764880.806541:0:24028:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f99703d0000 x1862022967067136/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1080/132280 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764880.806553:0:24028:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebc00/0x36f9bdc9e421cab7 lrc: 3/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 24028 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764880.806580:0:24028:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24028:1862022967067136:192.168.202.146@tcp:101:lfs.0 00000100:00100000:2.0:1775764880.806898:0:24028:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9966b46240 going to sleep for 11 seconds 00000100:00100000:2.0:1775764880.813171:0:24028:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24028:1862022967067136:192.168.202.146@tcp:101:lfs.0 00010000:00010000:2.0:1775764880.813195:0:24028:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebc00/0x36f9bdc9e421cab7 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 24028 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764880.813209:0:24028:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebc00/0x36f9bdc9e421cab7 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 24028 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764880.813216:0:24028:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebc00/0x36f9bdc9e421cab7 lrc: 4/1,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 24028 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764880.813223:0:24028:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebc00/0x36f9bdc9e421cab7 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 24028 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764880.813231:0:24028:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebc00/0x36f9bdc9e421cab7 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 24028 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764880.813282:0:24028:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99706ebc00 already unused, calling callback (ffffffffc1b652e0) 00010000:00010000:2.0:1775764880.813287:0:24028:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebc00/0x36f9bdc9e421cab7 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 24028 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764880.813294:0:24028:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebc00/0x36f9bdc9e421cab7 lrc: 4/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 24028 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764880.813301:0:24028:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebc00/0x36f9bdc9e421cab7 lrc: 5/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 24028 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764880.813308:0:24028:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1775764880.813316:0:24028:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebc00/0x36f9bdc9e421cab7 lrc: 3/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 24028 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764880.813322:0:24028:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ebc00/0x36f9bdc9e421cab7 lrc: 0/0,0 mode: --/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 24028 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764880.813342:0:24028:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff8f99703d0000 x1862022967067136/t0(0) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1080/608 e 0 to 0 dl 1775764896 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764880.813357:0:24028:0:(namei.c:943:ll_intent_lock()) intent lock 1 on i1 [0x200000401:0xee2:0x0] suppgids 0 -1: rc -20 00000080:00200000:2.0:1775764880.813505:0:24028:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8),name=/ 00000002:00010000:2.0:1775764880.813525:0:24028:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764880.813539:0:24028:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00010000:00010000:2.0:1775764880.813551:0:24028:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00000080:00200000:2.0:1775764880.813558:0:24028:0:(namei.c:943:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764880.813578:0:24028:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) 00000080:00010000:2.0:1775764880.813583:0:24028:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8f9952d121c8) for lock 0x36f9bdc9e421ca24 00000080:00010000:2.0:1775764880.813588:0:24028:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x36f9bdc9e421ca24 from it ffffafbac1ccbbf8 00010000:00010000:2.0:1775764880.813592:0:24028:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00010000:00010000:2.0:1775764880.813609:0:24028:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f99706ea400/0x36f9bdc9e421ca24 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc522 expref: -99 pid: 23966 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00000080:00200000:2.0:1775764880.813626:0:24028:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8f9952d121c8), inode mode 41ed mask 1 00000080:00200000:2.0:1775764880.813635:0:24028:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d14.sanity-pfl, flags=267 00000080:00200000:2.0:1775764880.813643:0:24028:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000401:0xee2:0x0](ffff8f9952e3ba88), inode mode 41ed mask 44 00000080:00200008:2.0:1775764880.813651:0:24028:0:(file.c:1057:ll_file_open()) START file d14.sanity-pfl:[0x200000401:0xee2:0x0](ffff8f9952e3ba88), flags 304000 00000080:00200080:2.0:1775764880.813691:0:24028:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000401:0xee2:0x0](ffff8f9952e3ba88) cmd=c0086915 arg=20aa3d0 00000100:00100000:2.0:1775764880.813800:0:24028:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24028:1862022967067264:192.168.202.146@tcp:34:lfs.0 00000100:00100000:2.0:1775764880.813887:0:24028:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9966b46240 going to sleep for 11 seconds 00000100:00100000:2.0:1775764880.821104:0:24028:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job lfs:3f83546c-01a9-4156-8a97-44c446245d3e:24028:1862022967067264:192.168.202.146@tcp:34:lfs.0 00000080:00200008:2.0:1775764880.821261:0:24028:0:(file.c:447:ll_file_release()) START file d14.sanity-pfl:[0x200000401:0xee2:0x0](ffff8f9952e3ba88), flags 304000 00010000:00010000:2.0:1775764880.821299:0:24028:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d600/0x36f9bdc9e421ca9b lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x2aac6a50e5fc736 expref: -99 pid: 24024 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:3.0:1775764880.967630:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764881.992357:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764883.015408:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764883.144040:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2382:1862022967067392:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764883.144147:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967067648:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764883.144187:0:2382:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:3.0:1775764883.144193:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967067520:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764883.144216:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967067776:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764883.155270:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967067520:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764883.155288:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967067776:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764883.155299:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2382:1862022967067392:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764883.155312:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967067648:192.168.202.146@tcp:400:kworker.0 00010000:00010000:2.0:1775764884.041812:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764885.062290:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764886.105981:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1775764886.728173:0:38:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xee0:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764886.728189:0:38:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:1.0:1775764886.728249:0:38:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:1.0:1775764886.728269:0:38:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x10 00020000:00200000:1.0:1775764886.728283:0:38:0:(lov_io.c:860:lov_io_iter_init()) component[3] flags 0x10 00020000:00200000:1.0:1775764886.728296:0:38:0:(lov_io.c:860:lov_io_iter_init()) component[4] flags 0x0 00000008:00010000:1.0:1775764886.728315:0:38:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### extent: 000000003deacf37 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000587dd0e8/0x36f9bdc9e421ca0f lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc4ab expref: -99 pid: 22506 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:1.0:1775764886.728334:0:38:0:(osc_cache.c:1167:osc_extent_make_ready()) ### extent: 000000003deacf37 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000587dd0e8/0x36f9bdc9e421ca0f lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc4ab expref: -99 pid: 22506 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764886.728400:0:38:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (3145728 3149823) ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000587dd0e8/0x36f9bdc9e421ca0f lrc: 3/0,0 mode: PW/PW res: [0x240000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc4ab expref: -99 pid: 22506 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:1.0:1775764886.728476:0:38:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8f994f2e4a80 - object 0x240000403:1796 offset 3145728<>3145729 00000080:00200000:1.0:1775764886.728567:0:38:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xee0:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:1.0:1775764886.728607:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967067904:192.168.202.146@tcp:4:rwv.0 00000008:00010000:0.0:1775764886.735154:0:2380:0:(osc_cache.c:901:osc_extent_finish()) ### extent: 000000003deacf37 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000587dd0e8/0x36f9bdc9e421ca0f lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc4ab expref: -99 pid: 22506 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1775764886.735246:0:2380:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000003deacf37 ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000587dd0e8/0x36f9bdc9e421ca0f lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x800020000000000 nid: local remote: 0x2aac6a50e5fc4ab expref: -99 pid: 22506 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1775764886.735278:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967067904:192.168.202.146@tcp:4:rwv.0 00010000:00010000:3.0:1775764887.110112:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764888.133986:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764888.263152:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:884ae1b0-72f8-4a88-a129-db99cded7efc:2379:1862022967068032:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764888.263258:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967068288:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764888.263292:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:0.0:1775764888.263299:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967068160:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764888.263320:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967068416:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764888.302444:0:2379:0:(client.c:3082:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8f99523ee800: committing for last_committed 4294971568 gen 5 00000100:00100000:0.0:1775764888.302484:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967068288:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764888.302511:0:2379:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8f99523ee800: committing for last_committed 8589934642 gen 1 00000100:00100000:0.0:1775764888.302586:0:2379:0:(client.c:3131:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff8f994f2e5880 x1862022967065600/t8589934642(8589934642) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1072/608 e 0 to 0 dl 1775764896 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1775764888.302599:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967068160:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764888.302617:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967068416:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764888.302631:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:884ae1b0-72f8-4a88-a129-db99cded7efc:2379:1862022967068032:192.168.202.146@tcp:400:kworker.0 00010000:00010000:3.0:1775764889.157939:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764890.084584:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:1.0:1775764890.084634:0:2371:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.146@tcp (source: 12345-192.168.202.146@tcp) 00000100:00100000:0.0:1775764890.084887:0:11131:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978436736 00000100:00100000:0.0:1775764890.084900:0:11131:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978436736 00000100:00100000:0.0:1775764890.084960:0:11131:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022978436864 00000100:00100000:0.0:1775764890.084963:0:11131:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022978436864 00000100:00100000:0.0:1775764890.084986:0:11131:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24234 00000100:00100000:0.0:1775764890.084996:0:11131:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e7480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:884ae1b0-72f8-4a88-a129-db99cded7efc+4:19386:x1862022978436736:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764890.085015:0:11131:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.202.146@tcp lock: ffff8f99821cce00/0x36f9bdc9e421ca55 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1400000000000 nid: local remote: 0x2aac6a50e5fc5f4 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764890.085102:0:11131:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1775764890.085109:0:11131:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e7480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:884ae1b0-72f8-4a88-a129-db99cded7efc+4:19386:x1862022978436736:12345-192.168.202.146@tcp:104: Request processed in 115us (537us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764890.085118:0:11131:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24234 00000100:00100000:0.0:1775764890.085135:0:11131:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.146@tcp, seq: 24235 00000100:00100000:0.0:1775764890.085142:0:11131:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8f994f2e4380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:884ae1b0-72f8-4a88-a129-db99cded7efc+3:19386:x1862022978436864:12345-192.168.202.146@tcp:104: 00010000:00010000:0.0:1775764890.085150:0:11131:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.202.146@tcp lock: ffff8f996695c000/0x36f9bdc9e421ca4e lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1400000000000 nid: local remote: 0x2aac6a50e5fc5ed expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764890.085187:0:11131:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:0.0:1775764890.085192:0:11131:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8f994f2e4380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:884ae1b0-72f8-4a88-a129-db99cded7efc+3:19386:x1862022978436864:12345-192.168.202.146@tcp:104: Request processed in 51us (558us total) trans 0 rc 0/0 00000100:00100000:0.0:1775764890.085197:0:11131:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.146@tcp, seq: 24235 00010000:00010000:3.0:1775764890.085630:0:11210:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1775764890.085637:0:11210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.146@tcp lock: ffff8f99821cce00/0x36f9bdc9e421ca55 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1400000000000 nid: local remote: 0x2aac6a50e5fc5f4 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764890.085645:0:11210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f99821cce00 already unused, calling callback (ffffffffc1393c20) 10000000:00010000:3.0:1775764890.085649:0:11210:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.146@tcp lock: ffff8f99821cce00/0x36f9bdc9e421ca55 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1400400000000 nid: local remote: 0x2aac6a50e5fc5f4 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764890.085682:0:11210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.146@tcp lock: ffff8f99821cce00/0x36f9bdc9e421ca55 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1408400000000 nid: local remote: 0x2aac6a50e5fc5f4 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:3.0:1775764890.085690:0:11210:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.146@tcp lock: ffff8f99821cce00/0x36f9bdc9e421ca55 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1409400000000 nid: local remote: 0x2aac6a50e5fc5f4 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1775764890.085697:0:11210:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:3.0:1775764890.085771:0:11210:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: MGC192.168.202.146@tcp lock: ffff8f99821cce00/0x36f9bdc9e421ca55 lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5c09400000020 nid: local remote: 0x2aac6a50e5fc5f4 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764890.085778:0:11210:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764890.085797:0:11210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.146@tcp lock: ffff8f99821cce00/0x36f9bdc9e421ca55 lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5c09400000020 nid: local remote: 0x2aac6a50e5fc5f4 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764890.085804:0:11210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.146@tcp lock: ffff8f99821cce00/0x36f9bdc9e421ca55 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5c09400000020 nid: local remote: 0x2aac6a50e5fc5f4 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764890.085817:0:11210:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1775764890.085819:0:11210:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.202.146@tcp lock: ffff8f996695c000/0x36f9bdc9e421ca4e lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0x2aac6a50e5fc5ed expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764890.085825:0:11210:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8f996695c000 already unused, calling callback (ffffffffc1393c20) 10000000:00010000:3.0:1775764890.085827:0:11210:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.202.146@tcp lock: ffff8f996695c000/0x36f9bdc9e421ca4e lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1400400000000 nid: local remote: 0x2aac6a50e5fc5ed expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764890.085833:0:11210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.202.146@tcp lock: ffff8f996695c000/0x36f9bdc9e421ca4e lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1408400000000 nid: local remote: 0x2aac6a50e5fc5ed expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:3.0:1775764890.085850:0:11210:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.202.146@tcp lock: ffff8f996695c000/0x36f9bdc9e421ca4e lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1409400000000 nid: local remote: 0x2aac6a50e5fc5ed expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1775764890.085869:0:11210:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:3.0:1775764890.085882:0:11210:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: MGC192.168.202.146@tcp lock: ffff8f996695c000/0x36f9bdc9e421ca4e lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x2aac6a50e5fc5ed expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764890.085887:0:11210:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1775764890.085893:0:11210:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.202.146@tcp lock: ffff8f996695c000/0x36f9bdc9e421ca4e lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x2aac6a50e5fc5ed expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764890.085899:0:11210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.202.146@tcp lock: ffff8f996695c000/0x36f9bdc9e421ca4e lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x2aac6a50e5fc5ed expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764890.085912:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764890.085914:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764890.085916:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764890.085931:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2382:1862022967068544:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764890.086001:0:2382:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1775764890.086008:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2382:1862022967068672:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1775764890.086140:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764890.094428:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2382:1862022967068672:192.168.202.146@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1775764890.094446:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2382:1862022967068544:192.168.202.146@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1775764890.181927:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764891.205961:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:01000000:0.0:1775764891.269877:0:5575:0:(mgc_request.c:611:do_requeue()) updating log lustre-client 10000000:01000000:0.0:1775764891.269885:0:5575:0:(mgc_request.c:1819:mgc_process_log()) Process log lustre-client-ffff8f99523ee800 from 48 10000000:01000000:0.0:1775764891.269892:0:5575:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-client (res 0x65727473756c) 00010000:00010000:0.0:1775764891.269946:0:5575:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.146@tcp lock: ffff8f99821ccc00/0x36f9bdc9e421cabe lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764891.269958:0:5575:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.146@tcp lock: ffff8f99821ccc00/0x36f9bdc9e421cabe lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1775764891.269970:0:5575:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.146@tcp lock: ffff8f99821ccc00/0x36f9bdc9e421cabe lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1775764891.269999:0:5575:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967068800:192.168.202.146@tcp:101:ll_cfg_requeue.0 00000100:00100000:0.0:1775764891.270126:0:5575:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994ecbbc00 going to sleep for 11 seconds 00000100:00100000:3.0:1775764891.275782:0:5575:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967068800:192.168.202.146@tcp:101:ll_cfg_requeue.0 00010000:00010000:3.0:1775764891.275811:0:5575:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.202.146@tcp lock: ffff8f99821ccc00/0x36f9bdc9e421cabe lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x2aac6a50e5fc86a expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764891.275874:0:5575:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.202.146@tcp lock: ffff8f99821ccc00/0x36f9bdc9e421cabe lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x2aac6a50e5fc86a expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764891.275951:0:5575:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967068928:192.168.202.146@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1775764891.276031:0:5575:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9966b46540 going to sleep for 11 seconds 00000100:00100000:3.0:1775764891.278308:0:5575:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967068928:192.168.202.146@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1775764891.278368:0:5575:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967069056:192.168.202.146@tcp:503:ll_cfg_requeue.0 00000100:00100000:3.0:1775764891.278469:0:5575:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9966b46540 going to sleep for 11 seconds 00000100:00100000:3.0:1775764891.285879:0:5575:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967069056:192.168.202.146@tcp:503:ll_cfg_requeue.0 00000100:00100000:0.0:1775764891.287262:0:24489:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job llog_process_th:884ae1b0-72f8-4a88-a129-db99cded7efc:24489:1862022967069184:192.168.202.146@tcp:502:llog_process_th.0 00000100:00100000:0.0:1775764891.287375:0:24489:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f994ecbb540 going to sleep for 11 seconds 00000100:00100000:2.0:1775764891.293878:0:24489:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job llog_process_th:884ae1b0-72f8-4a88-a129-db99cded7efc:24489:1862022967069184:192.168.202.146@tcp:502:llog_process_th.0 00000020:01000000:2.0:1775764891.293926:0:24489:0:(obd_config.c:1888:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:2.0:1775764891.293959:0:24489:0:(obd_config.c:1468:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1775764891.293962:0:24489:0:(obd_config.c:1559:class_process_config()) marker 32 (0x1) lustre-clilov del lustre.pool1 00000020:01000000:2.0:1775764891.293970:0:24489:0:(obd_config.c:2003:class_config_llog_handler()) cmd ce023, instance name: lustre-clilov-ffff8f99523ee800 00000020:00000080:2.0:1775764891.293973:0:24489:0:(obd_config.c:1468:class_process_config()) processing cmd: ce023 00000100:00100000:2.0:1775764891.294080:0:24489:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job llog_process_th:884ae1b0-72f8-4a88-a129-db99cded7efc:24489:1862022967069312:192.168.202.146@tcp:502:llog_process_th.0 00000100:00100000:2.0:1775764891.294178:0:24489:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703afc00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764891.300336:0:24489:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job llog_process_th:884ae1b0-72f8-4a88-a129-db99cded7efc:24489:1862022967069312:192.168.202.146@tcp:502:llog_process_th.0 00000020:01000000:2.0:1775764891.300378:0:24489:0:(obd_config.c:1888:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:2.0:1775764891.300388:0:24489:0:(obd_config.c:1468:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1775764891.300390:0:24489:0:(obd_config.c:1559:class_process_config()) marker 32 (0x2) lustre-clilov del lustre.pool1 00000040:00080000:2.0:1775764891.300394:0:24489:0:(llog.c:857:llog_process_thread()) stop processing plain [0xa:0x3:0x0] index 52 count 51 00000020:01000000:3.0:1775764891.301016:0:5575:0:(obd_config.c:2143:class_config_parse_llog()) Processed log lustre-client gen 48-51 (rc=0) 10000000:01000000:3.0:1775764891.301041:0:5575:0:(mgc_request.c:1896:mgc_process_log()) MGC192.168.202.146@tcp: configuration from log 'lustre-client' succeeded (0). 00010000:00010000:3.0:1775764891.301053:0:5575:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.146@tcp lock: ffff8f99821ccc00/0x36f9bdc9e421cabe lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x2aac6a50e5fc86a expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764891.301064:0:5575:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.202.146@tcp lock: ffff8f99821ccc00/0x36f9bdc9e421cabe lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x2aac6a50e5fc86a expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1775764891.301075:0:5575:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:01000000:3.0:1775764891.301078:0:5575:0:(mgc_request.c:1819:mgc_process_log()) Process log lustre-sptlrpc-ffff8f9942a1b380 from 1 10000000:01000000:3.0:1775764891.301081:0:5575:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00010000:00010000:3.0:1775764891.301119:0:5575:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.146@tcp lock: ffff8f99435d5a00/0x36f9bdc9e421cac5 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764891.301127:0:5575:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.146@tcp lock: ffff8f99435d5a00/0x36f9bdc9e421cac5 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764891.301286:0:5575:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.146@tcp lock: ffff8f99435d5a00/0x36f9bdc9e421cac5 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764891.301311:0:5575:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967069440:192.168.202.146@tcp:101:ll_cfg_requeue.0 00000100:00100000:3.0:1775764891.301693:0:5575:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9966b46540 going to sleep for 11 seconds 00000100:00100000:3.0:1775764891.305108:0:5575:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967069440:192.168.202.146@tcp:101:ll_cfg_requeue.0 00010000:00010000:3.0:1775764891.305264:0:5575:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.202.146@tcp lock: ffff8f99435d5a00/0x36f9bdc9e421cac5 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x2aac6a50e5fc871 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764891.305277:0:5575:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.202.146@tcp lock: ffff8f99435d5a00/0x36f9bdc9e421cac5 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x2aac6a50e5fc871 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1775764891.305396:0:5575:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967069568:192.168.202.146@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1775764891.305474:0:5575:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f9966b46540 going to sleep for 11 seconds 00000100:00100000:3.0:1775764891.312639:0:5575:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:884ae1b0-72f8-4a88-a129-db99cded7efc:5575:1862022967069568:192.168.202.146@tcp:501:ll_cfg_requeue.0 10000000:01000000:3.0:1775764891.312684:0:5575:0:(mgc_request.c:1896:mgc_process_log()) MGC192.168.202.146@tcp: configuration from log 'lustre-sptlrpc' failed (-2). 00010000:00010000:3.0:1775764891.312694:0:5575:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.146@tcp lock: ffff8f99435d5a00/0x36f9bdc9e421cac5 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x2aac6a50e5fc871 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1775764891.312706:0:5575:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.202.146@tcp lock: ffff8f99435d5a00/0x36f9bdc9e421cac5 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x2aac6a50e5fc871 expref: -99 pid: 5575 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764892.236519:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764893.253968:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764893.382066:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967069824:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764893.382201:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967070080:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764893.382276:0:2381:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1775764893.382283:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:884ae1b0-72f8-4a88-a129-db99cded7efc:2381:1862022967069696:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764893.382310:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967069952:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764893.387332:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967069824:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764893.395381:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:884ae1b0-72f8-4a88-a129-db99cded7efc:2381:1862022967069696:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764893.395405:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967070080:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764893.402281:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967069952:192.168.202.146@tcp:400:kworker.0 00010000:00010000:1.0:1775764894.286057:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764895.302100:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764896.325930:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764897.349906:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764898.374492:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764898.501971:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967070336:192.168.202.146@tcp:400:kworker.0 00000100:00100000:2.0:1775764898.501985:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2382:1862022967070208:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764898.502149:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967070592:192.168.202.146@tcp:400:kworker.0 00000100:00100000:2.0:1775764898.502293:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967070464:192.168.202.146@tcp:400:kworker.0 00010000:00010000:1.0:1775764899.398021:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764900.421935:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764901.446053:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764902.469880:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764903.493877:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764903.621945:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967070848:192.168.202.146@tcp:400:kworker.0 00000100:00100000:1.0:1775764903.621945:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:884ae1b0-72f8-4a88-a129-db99cded7efc:2379:1862022967070720:192.168.202.146@tcp:400:kworker.0 00000100:00100000:1.0:1775764903.622039:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967070976:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764903.622044:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967071104:192.168.202.146@tcp:400:kworker.0 00010000:00010000:2.0:1775764904.517919:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764905.541886:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764906.565967:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764907.589857:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764908.613872:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764908.742023:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2382:1862022967071232:192.168.202.146@tcp:400:kworker.0 00000100:00100000:2.0:1775764908.742198:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967071360:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764908.742280:0:2382:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967071488:192.168.202.146@tcp:400:kworker.0 00000100:00100000:2.0:1775764908.742430:0:2381:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967071616:192.168.202.146@tcp:400:kworker.0 00010000:00010000:3.0:1775764909.637838:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764910.661923:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764911.685885:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764912.709896:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764913.733897:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764913.861935:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967071872:192.168.202.146@tcp:400:kworker.0 00000100:00100000:1.0:1775764913.861942:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:884ae1b0-72f8-4a88-a129-db99cded7efc:2379:1862022967071744:192.168.202.146@tcp:400:kworker.0 00000100:00100000:1.0:1775764913.862013:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967072000:192.168.202.146@tcp:400:kworker.0 00000100:00100000:0.0:1775764913.862019:0:2380:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967072128:192.168.202.146@tcp:400:kworker.0 00010000:00010000:3.0:1775764914.757896:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:2.0:1775764914.885815:0:2381:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764898/real 1775764898] req@ffff8f9970edd500 x1862022967070336/t0(0) o400->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764914 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000400:3.0:1775764914.885815:0:2382:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764898/real 1775764898] req@ffff8f9970edc700 x1862022967070208/t0(0) o400->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 224/224 e 0 to 1 dl 1775764914 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764914.885881:0:2382:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8f9970edc700 x1862022967070208/t0(0) o400->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 224/224 e 0 to 1 dl 1775764914 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775764914.885888:0:2382:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from FULL to DISCONN 00000100:02020000:3.0:1775764914.885892:0:2382:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.202.146@tcp: Connection to MGS (at 192.168.202.146@tcp) was lost; in progress operations using this service will fail 00000100:00080000:3.0:1775764914.895174:0:2382:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.202.146@tcp for MGC192.168.202.146@tcp not replayable, auto-deactivating 00000100:00080000:3.0:1775764914.895177:0:2382:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00100000:3.0:1775764914.895183:0:2382:0:(client.c:3560:ptlrpc_abort_inflight()) @@@ inflight req@ffff8f9970edc700 x1862022967070208/t0(0) o400->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 224/224 e 0 to 1 dl 1775764914 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764914.895192:0:2382:0:(client.c:3560:ptlrpc_abort_inflight()) @@@ inflight req@ffff8f9970a77100 x1862022967070720/t0(0) o400->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 224/224 e 0 to 0 dl 1775764919 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764914.895202:0:2382:0:(client.c:3560:ptlrpc_abort_inflight()) @@@ inflight req@ffff8f9970ede680 x1862022967071232/t0(0) o400->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 224/224 e 0 to 0 dl 1775764924 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764914.895208:0:2382:0:(client.c:3560:ptlrpc_abort_inflight()) @@@ inflight req@ffff8f994f2e4700 x1862022967071744/t0(0) o400->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 224/224 e 0 to 0 dl 1775764929 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 10000000:01000000:3.0:1775764914.895215:0:2382:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1775764914.895217:0:2382:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:01000000:3.0:1775764914.895219:0:2382:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 00000100:00080000:3.0:1775764914.895220:0:2382:0:(import.c:416:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000400:3.0:1775764914.895234:0:2382:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764898/real 1775764898] req@ffff8f9970edea00 x1862022967070464/t0(0) o400->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 224/224 e 0 to 1 dl 1775764914 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764914.895244:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:884ae1b0-72f8-4a88-a129-db99cded7efc:2379:1862022967071744:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764914.895249:0:2382:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8f9970edea00 x1862022967070464/t0(0) o400->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 224/224 e 0 to 1 dl 1775764914 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775764914.895255:0:2382:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from FULL to DISCONN 00000100:00100000:1.0:1775764914.895259:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:884ae1b0-72f8-4a88-a129-db99cded7efc:2379:1862022967070720:192.168.202.146@tcp:400:kworker.0 00000100:02000400:3.0:1775764914.895259:0:2382:0:(import.c:181:ptlrpc_set_import_discon()) lustre-OST0000-osc-ffff8f99523ee800: Connection to lustre-OST0000 (at 192.168.202.146@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:1.0:1775764914.895311:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764914.895316:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775764914.895318:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775764914.895323:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1438 00000100:00080000:1.0:1775764914.895334:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775764914.895361:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967072256/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764914.895481:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967072256:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775764914.895654:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967072384:192.168.202.146@tcp:400:kworker.0 00000100:00100000:2.0:1775764914.897504:0:2381:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8f9970edd500 x1862022967070336/t0(0) o400->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764914 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764914.897525:0:2381:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:2.0:1775764914.897529:0:2381:0:(import.c:181:ptlrpc_set_import_discon()) lustre-MDT0000-mdc-ffff8f99523ee800: Connection to lustre-MDT0000 (at 192.168.202.146@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:2.0:1775764914.897532:0:2381:0:(import.c:416:ptlrpc_pinger_force()) lustre-MDT0000_UUID: waking up pinger s:DISCONN 00000100:00000400:2.0:1775764914.897541:0:2381:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764898/real 1775764898] req@ffff8f9970edc000 x1862022967070592/t0(0) o400->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 224/224 e 0 to 1 dl 1775764914 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764914.897551:0:2381:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8f9970edc000 x1862022967070592/t0(0) o400->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 224/224 e 0 to 1 dl 1775764914 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764914.897556:0:2381:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from FULL to DISCONN 00000100:02000400:2.0:1775764914.897558:0:2381:0:(import.c:181:ptlrpc_set_import_discon()) lustre-OST0001-osc-ffff8f99523ee800: Connection to lustre-OST0001 (at 192.168.202.146@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:2.0:1775764914.897562:0:2381:0:(import.c:416:ptlrpc_pinger_force()) lustre-OST0001_UUID: waking up pinger s:DISCONN 00000100:00100000:2.0:1775764914.897579:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967070336:192.168.202.146@tcp:400:kworker.0 00000100:00080000:3.0:1775764914.902540:0:2382:0:(import.c:416:ptlrpc_pinger_force()) lustre-OST0000_UUID: waking up pinger s:DISCONN 00000100:00080000:1.0:1775764914.902552:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764914.902558:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775764914.902561:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775764914.902569:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 577 00000100:00100000:3.0:1775764914.902574:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2382:1862022967071232:192.168.202.146@tcp:400:kworker.0 00000100:00080000:1.0:1775764914.902581:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775764914.902583:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:884ae1b0-72f8-4a88-a129-db99cded7efc:2382:1862022967070208:192.168.202.146@tcp:400:kworker.0 00000008:00100000:1.0:1775764914.902591:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775764914.902609:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967072512/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775764914.902628:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764914.902630:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775764914.902631:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775764914.902634:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 577 00000100:00080000:1.0:1775764914.902638:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775764914.902641:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775764914.902649:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967072640/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764914.902653:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967072512:192.168.202.146@tcp:8:kworker.0 00000100:00080000:1.0:1775764914.902667:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764914.902670:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775764914.902671:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775764914.902674:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1448 00000100:00080000:1.0:1775764914.902678:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775764914.902680:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775764914.902703:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967072768/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764914.902710:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967072640:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775764914.902767:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967072768:192.168.202.146@tcp:38:kworker.0 00010000:00010000:3.0:1775764915.781828:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764916.805857:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1775764917.445909:0:38:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xee0:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764917.445924:0:38:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:2.0:1775764917.445943:0:38:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:2.0:1775764917.445955:0:38:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x10 00020000:00200000:2.0:1775764917.445968:0:38:0:(lov_io.c:860:lov_io_iter_init()) component[3] flags 0x10 00020000:00200000:2.0:1775764917.445980:0:38:0:(lov_io.c:860:lov_io_iter_init()) component[4] flags 0x0 00000080:00200000:2.0:1775764917.446031:0:38:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xee0:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:1.0:1775764917.829886:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764918.853878:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764919.877885:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775764920.005844:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000400:1.0:1775764920.005853:0:2379:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764903/real 1775764903] req@ffff8f9970a77480 x1862022967070976/t0(0) o400->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 224/224 e 0 to 1 dl 1775764919 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775764920.005854:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1775764920.005858:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764920.005866:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1775764920.005872:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764920.005876:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1775764920.005879:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764920.005882:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1775764920.005894:0:2379:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8f9970a77480 x1862022967070976/t0(0) o400->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 224/224 e 0 to 1 dl 1775764919 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764920.005904:0:2379:0:(import.c:213:ptlrpc_set_import_discon()) osc: import ffff8f9952a6d800 already not connected (conn 6, was 5): CONNECTING 00000100:00000400:1.0:1775764920.005932:0:2380:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764903/real 1775764903] req@ffff8f9970a76300 x1862022967070848/t0(0) o400->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764919 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764920.005945:0:2380:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8f9970a76300 x1862022967070848/t0(0) o400->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764919 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764920.005950:0:2380:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff8f9952a6b000 already not connected (conn 4, was 3): CONNECTING 00000100:00000400:1.0:1775764920.005955:0:2380:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764903/real 1775764903] req@ffff8f9970a77800 x1862022967071104/t0(0) o400->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 224/224 e 0 to 1 dl 1775764919 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764920.005965:0:2380:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8f9970a77800 x1862022967071104/t0(0) o400->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 224/224 e 0 to 1 dl 1775764919 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764920.005969:0:2380:0:(import.c:213:ptlrpc_set_import_discon()) osc: import ffff8f9952a6f000 already not connected (conn 7, was 6): CONNECTING 00000100:00100000:1.0:1775764920.005974:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967070848:192.168.202.146@tcp:400:kworker.0 00010000:00010000:3.0:1775764920.901845:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764921.925871:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764922.949890:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764923.973886:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764924.101828:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764914/real 1775764914] req@ffff8f994f2e4a80 x1862022967072768/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764924 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000400:2.0:1775764924.101852:0:2381:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764908/real 1775764908] req@ffff8f9970edce00 x1862022967071360/t0(0) o400->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764924 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000400:3.0:1775764924.101855:0:2382:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764908/real 1775764908] req@ffff8f9970edfb80 x1862022967071488/t0(0) o400->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 224/224 e 0 to 1 dl 1775764924 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764924.101860:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967072768/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764924 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764924.101897:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764914/real 1775764914] req@ffff8f994f2e5f80 x1862022967072640/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775764924 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775764924.101901:0:2382:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8f9970edfb80 x1862022967071488/t0(0) o400->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 224/224 e 0 to 1 dl 1775764924 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775764924.101919:0:2382:0:(import.c:213:ptlrpc_set_import_discon()) osc: import ffff8f9952a6d800 already not connected (conn 6, was 5): CONNECTING 00000100:00100000:1.0:1775764924.101927:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967072640/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775764924 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764924.101938:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764914/real 1775764914] req@ffff8f994f2e6680 x1862022967072512/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775764924 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764924.101949:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967072512/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775764924 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764924.101955:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764914/real 1775764914] req@ffff8f994f2e4700 x1862022967072256/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764924 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764924.101962:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967072256/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764924 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764924.101970:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764924.101985:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775764924.101988:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967072768:192.168.202.146@tcp:38:kworker.0 00000100:00080000:1.0:1775764924.101998:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764924.102004:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775764924.102008:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967072640:192.168.202.146@tcp:8:kworker.0 00000100:00080000:3.0:1775764924.102020:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764924.102026:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:0.0:1775764924.102038:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967071104:192.168.202.146@tcp:400:kworker.0 00000100:00080000:3.0:1775764924.102049:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764924.102052:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775764924.102054:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764924.102058:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1541 00000100:00080000:1.0:1775764924.102063:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775764924.102067:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 5s 00000100:00080000:1.0:1775764924.102075:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00080000:3.0:1775764924.102075:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775764924.102079:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967070976:192.168.202.146@tcp:400:kworker.0 00000100:00100000:1.0:1775764924.102082:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967072512:192.168.202.146@tcp:8:kworker.0 00000100:00080000:1.0:1775764924.102086:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1775764924.102089:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1775764924.102092:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775764924.102096:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967072256:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775764924.102120:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967072896/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764924.102135:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764924.102137:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775764924.102139:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764924.102142:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1541 00000100:00080000:3.0:1775764924.102144:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1775764924.102147:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775764924.102151:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775764924.102153:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967072896:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775764924.102159:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967073024/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764924.102166:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764924.102168:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775764924.102169:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764924.102170:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1541 00000100:00080000:3.0:1775764924.102172:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1775764924.102184:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775764924.102187:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764924.102197:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967073152/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764924.102234:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764924.102238:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775764924.102240:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775764924.102244:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967073024:192.168.202.146@tcp:8:kworker.0 00000100:00080000:3.0:1775764924.102245:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1541 00000100:00080000:3.0:1775764924.102253:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1775764924.102257:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775764924.102267:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967073280/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764924.102282:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967073152:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775764924.102287:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967071488:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764924.102292:0:2382:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:3f83546c-01a9-4156-8a97-44c446245d3e:2382:1862022967070464:192.168.202.146@tcp:400:kworker.0 00000100:00100000:1.0:1775764924.102310:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967073280:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775764924.116535:0:2381:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8f9970edce00 x1862022967071360/t0(0) o400->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764924 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764924.116544:0:2381:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff8f9952a6b000 already not connected (conn 5, was 3): CONNECTING 00000100:00000400:2.0:1775764924.116550:0:2381:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764908/real 1775764908] req@ffff8f9970eded80 x1862022967071616/t0(0) o400->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 224/224 e 0 to 1 dl 1775764924 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764924.116558:0:2381:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8f9970eded80 x1862022967071616/t0(0) o400->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 224/224 e 0 to 1 dl 1775764924 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764924.116562:0:2381:0:(import.c:213:ptlrpc_set_import_discon()) osc: import ffff8f9952a6f000 already not connected (conn 8, was 6): CONNECTING 00000100:00100000:2.0:1775764924.116573:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967071360:192.168.202.146@tcp:400:kworker.0 00000100:00100000:2.0:1775764924.116579:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967070592:192.168.202.146@tcp:400:kworker.0 00010000:00010000:1.0:1775764924.997859:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764926.021875:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764927.045881:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764928.069835:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764929.093882:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:0.0:1775764929.157860:0:2379:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764913/real 1775764913] req@ffff8f994f2e4e00 x1862022967072000/t0(0) o400->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 224/224 e 0 to 1 dl 1775764929 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000400:1.0:1775764929.157860:0:2380:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764913/real 1775764913] req@ffff8f994f2e7b80 x1862022967071872/t0(0) o400->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764929 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764929.157893:0:2380:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8f994f2e7b80 x1862022967071872/t0(0) o400->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764929 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764929.157901:0:2379:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8f994f2e4e00 x1862022967072000/t0(0) o400->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 224/224 e 0 to 1 dl 1775764929 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764929.157901:0:2380:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff8f9952a6b000 already not connected (conn 5, was 3): CONNECTING 00000100:00000400:1.0:1775764929.157911:0:2380:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764913/real 1775764913] req@ffff8f994f2e6300 x1862022967072128/t0(0) o400->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 224/224 e 0 to 1 dl 1775764929 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764929.157917:0:2379:0:(import.c:213:ptlrpc_set_import_discon()) osc: import ffff8f9952a6d800 already not connected (conn 7, was 5): CONNECTING 00000100:00100000:1.0:1775764929.157927:0:2380:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8f994f2e6300 x1862022967072128/t0(0) o400->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 224/224 e 0 to 1 dl 1775764929 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764929.157948:0:2380:0:(import.c:213:ptlrpc_set_import_discon()) osc: import ffff8f9952a6f000 already not connected (conn 8, was 6): CONNECTING 00000100:00100000:1.0:1775764929.157956:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967071872:192.168.202.146@tcp:400:kworker.0 00000100:00080000:3.0:1775764929.221830:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764929.221843:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1775764929.221845:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764929.221848:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1775764929.221850:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764929.221852:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1775764929.221870:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764929.221872:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:1.0:1775764930.117893:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:0.0:1775764930.181843:0:2379:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764914/real 1775764914] req@ffff8f994f2e7100 x1862022967072384/t0(0) o400->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764930 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764930.181869:0:2379:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8f994f2e7100 x1862022967072384/t0(0) o400->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764930 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764930.181875:0:2379:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff8f9952a6b000 already not connected (conn 5, was 3): CONNECTING 00000100:00100000:0.0:1775764930.181887:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967072384:192.168.202.146@tcp:400:kworker.0 00010000:00010000:1.0:1775764931.141916:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764932.165899:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764933.189873:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764934.213873:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775764934.341821:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764934.341828:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1775764934.341830:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764934.341833:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1775764934.341835:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764934.341836:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1775764934.341838:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764934.341839:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:1.0:1775764935.237896:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764936.261869:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764937.285909:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764938.309856:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764939.333900:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775764939.461821:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775764939.461826:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764924/real 1775764924] req@ffff8f9970ede300 x1862022967073280/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764939 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775764939.461827:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1775764939.461837:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764939.461840:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1775764939.461842:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764939.461844:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1775764939.461846:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764939.461848:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1775764939.461853:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967073280/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764939 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764939.461861:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764924/real 1775764924] req@ffff8f9970ede680 x1862022967073024/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775764939 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764939.461868:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967073024/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775764939 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764939.461873:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764924/real 1775764924] req@ffff8f9970edf800 x1862022967073152/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775764939 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764939.461879:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967073152/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775764939 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764939.461884:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764924/real 1775764924] req@ffff8f9970edc700 x1862022967072896/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764939 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764939.461891:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967072896/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764939 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764939.461920:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764939.461932:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775764939.461935:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967073280:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775764939.461948:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764939.461952:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775764939.461955:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967073024:192.168.202.146@tcp:8:kworker.0 00000100:00080000:3.0:1775764939.461965:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764939.461970:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1775764939.461979:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:3f83546c-01a9-4156-8a97-44c446245d3e:2379:1862022967072000:192.168.202.146@tcp:400:kworker.0 00000100:00080000:1.0:1775764939.461981:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775764939.461988:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967073152:192.168.202.146@tcp:8:kworker.0 00000100:00080000:3.0:1775764939.461988:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775764939.461993:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775764939.461995:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775764939.462002:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1550 00000100:00080000:1.0:1775764939.462005:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775764939.462006:0:2381:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:3f83546c-01a9-4156-8a97-44c446245d3e:2381:1862022967071616:192.168.202.146@tcp:400:kworker.0 00000100:00100000:1.0:1775764939.462010:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967072896:192.168.202.146@tcp:38:kworker.0 00000100:00080000:3.0:1775764939.462014:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 10s 00000100:00080000:3.0:1775764939.462024:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775764939.462033:0:2380:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:3f83546c-01a9-4156-8a97-44c446245d3e:2380:1862022967072128:192.168.202.146@tcp:400:kworker.0 00000100:00100000:3.0:1775764939.462052:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967073408/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764939.462081:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764939.462084:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775764939.462085:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764939.462088:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1550 00000100:00080000:3.0:1775764939.462091:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 10s 00000100:00080000:3.0:1775764939.462095:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775764939.462101:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775764939.462105:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967073408:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775764939.462112:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967073536/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764939.462119:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764939.462122:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775764939.462123:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764939.462125:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1550 00000100:00080000:3.0:1775764939.462128:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 10s 00000100:00080000:3.0:1775764939.462130:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775764939.462133:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764939.462139:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967073664/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764939.462155:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764939.462157:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775764939.462158:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764939.462161:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1550 00000100:00080000:3.0:1775764939.462164:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 10s 00000100:00080000:3.0:1775764939.462166:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775764939.462169:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764939.462178:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967073792/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764939.462181:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967073536:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775764939.462210:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967073664:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775764939.462231:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967073792:192.168.202.146@tcp:8:kworker.0 00010000:00010000:3.0:1775764940.357924:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764941.381885:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764942.405916:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764943.429924:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764944.453873:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775764944.581880:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775764944.581887:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1775764944.581890:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775764944.581898:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1775764944.581901:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775764944.581903:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1775764944.581906:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775764944.581908:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:2.0:1775764945.477909:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764946.501857:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764947.525956:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764948.549891:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000100:2.0:1775764948.677929:0:2370:0:(socklnd_cb.c:2376:ksocknal_find_timed_out_conn()) A connection with 12345-192.168.202.146@tcp (192.168.202.146:988) timed out; the network or node may be down. 00010000:00010000:1.0:1775764949.573927:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775764949.701901:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775764949.701907:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1775764949.701909:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775764949.701911:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1775764949.701913:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775764949.701914:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1775764949.701916:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775764949.701918:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:2.0:1775764950.597968:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764951.621986:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764952.645867:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764953.669976:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764954.693964:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775764954.821840:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764954.821850:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1775764954.821853:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764954.821856:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1775764954.821859:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764954.821862:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1775764954.821864:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764954.821866:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:1.0:1775764955.717969:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764956.741997:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764957.765918:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764958.789836:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764959.813859:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764959.941963:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764939/real 1775764939] req@ffff8f9970edd880 x1862022967073792/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775764959 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764959.942000:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967073792/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775764959 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764959.942014:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764939/real 1775764939] req@ffff8f9970edfb80 x1862022967073536/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764959 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764959.942026:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967073536/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764959 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764959.942036:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764939/real 1775764939] req@ffff8f9970edc380 x1862022967073664/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775764959 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764959.942046:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967073664/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775764959 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764959.942056:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764939/real 1775764939] req@ffff8f9970edea00 x1862022967073408/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764959 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764959.942066:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967073408/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764959 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764959.942085:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764959.942108:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775764959.942155:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967073792:192.168.202.146@tcp:8:kworker.0 00000100:00080000:3.0:1775764959.942155:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764959.942165:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1775764959.942166:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775764959.942173:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764959.942178:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775764959.942181:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764959.942188:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1566 00000100:00080000:3.0:1775764959.942199:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 15s 00000100:00080000:3.0:1775764959.942206:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775764959.942218:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1775764959.942228:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775764959.942233:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967073536:192.168.202.146@tcp:38:kworker.0 00000100:00080000:1.0:1775764959.942238:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1775764959.942289:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd180 x1862022967073920/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764959.942305:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764959.942309:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775764959.942311:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764959.942315:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1566 00000100:00080000:3.0:1775764959.942320:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 15s 00000100:00080000:3.0:1775764959.942324:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775764959.942329:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764959.942339:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddf80 x1862022967074048/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764959.942347:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764959.942350:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775764959.942351:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764959.942374:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1566 00000100:00080000:3.0:1775764959.942378:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 15s 00000100:00080000:3.0:1775764959.942380:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775764959.942384:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764959.942394:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967074176/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775764959.942407:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00080000:3.0:1775764959.942408:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764959.942413:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1775764959.942416:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967073664:192.168.202.146@tcp:8:kworker.0 00000100:00080000:1.0:1775764959.942423:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764959.942435:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775764959.942439:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967073408:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775764959.942446:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764959.942450:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775764959.942452:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764959.942457:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1566 00000100:00080000:3.0:1775764959.942462:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 15s 00000100:00080000:3.0:1775764959.942465:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775764959.942915:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967073920:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775764959.942923:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967074304/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764959.942988:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967074048:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775764959.943006:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967074176:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775764959.943023:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967074304:192.168.202.146@tcp:250:kworker.0 00010000:00010000:1.0:1775764960.837999:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764961.861874:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764962.885905:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1775764963.014167:0:2367:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.202.146:988 00000400:00000100:0.0:1775764963.014234:0:2367:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1775764963.014255:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775764963.014309:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775764963.014328:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:1.0:1775764963.014331:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764959/real 1775764963] req@ffff8f9970edd180 x1862022967073920/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:0.0:1775764963.014345:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:1.0:1775764963.014366:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd180 x1862022967073920/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764963.014375:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764963.014416:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775764963.014419:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967073920:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775764963.014424:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764959/real 1775764963] req@ffff8f9970eddf80 x1862022967074048/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775764984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764963.014438:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddf80 x1862022967074048/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775764984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764963.014443:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764963.014445:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775764963.014447:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967074048:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775764963.014451:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764959/real 1775764963] req@ffff8f9970edca80 x1862022967074176/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775764984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764963.014457:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967074176/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775764984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764963.014461:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764963.014463:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775764963.014465:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967074176:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775764963.014481:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764959/real 1775764963] req@ffff8f9970edf480 x1862022967074304/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764963.014487:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967074304/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764963.014491:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764963.014502:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775764963.014504:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967074304:192.168.202.146@tcp:250:kworker.0 00010000:00010000:2.0:1775764963.909892:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764964.933903:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775764965.062527:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764965.062538:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775764965.062543:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764965.062554:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1586 00000100:00080000:3.0:1775764965.062567:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764965.062576:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775764965.062624:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967074432/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764965.062980:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967074432:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775764965.063117:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764965.063122:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775764965.063125:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764965.063144:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1586 00000100:00080000:3.0:1775764965.063150:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764965.063158:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775764965.063169:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764965.063198:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967074560/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764965.063350:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967074560:192.168.202.146@tcp:38:kworker.0 00000100:00080000:3.0:1775764965.063424:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764965.063429:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775764965.063430:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764965.063435:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1586 00000100:00080000:3.0:1775764965.063440:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764965.063443:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775764965.063448:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764965.063461:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddf80 x1862022967074688/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764965.063470:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764965.063473:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775764965.063474:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764965.063477:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1586 00000100:00080000:3.0:1775764965.063508:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764965.063512:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775764965.063516:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764965.063527:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd180 x1862022967074816/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764965.063592:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967074688:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775764965.063613:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967074816:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775764965.957908:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1775764966.086281:0:2369:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.202.146:988 00000400:00000100:0.0:1775764966.086361:0:2369:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1775764966.086390:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775764966.086448:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775764966.086461:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775764966.086471:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:2.0:1775764966.086554:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764965/real 1775764966] req@ffff8f9970eddf80 x1862022967074688/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775764990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764966.086599:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddf80 x1862022967074688/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775764990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764966.086616:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764966.086624:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775764966.086634:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967074688:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775764966.086645:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764965/real 1775764966] req@ffff8f9970edd180 x1862022967074816/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775764990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764966.086660:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd180 x1862022967074816/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775764990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764966.086667:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764966.086671:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775764966.086674:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967074816:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775764966.086682:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764965/real 1775764966] req@ffff8f9970edca80 x1862022967074560/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764966.086694:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967074560/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764966.086702:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764966.087270:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775764966.087280:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967074560:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775764966.087294:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764965/real 1775764966] req@ffff8f9970edf480 x1862022967074432/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764966.087324:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967074432/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764966.087335:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764966.087338:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775764966.087342:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967074432:192.168.202.146@tcp:250:kworker.0 00010000:00010000:1.0:1775764966.981931:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764968.005890:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764969.029868:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764970.053873:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775764970.182176:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775764970.182186:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775764970.182191:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775764970.182241:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1591 00000100:00080000:0.0:1775764970.182253:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775764970.182259:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775764970.182293:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967074944/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775764970.182335:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775764970.182338:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775764970.182340:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775764970.182345:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1591 00000100:00080000:0.0:1775764970.182348:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775764970.182351:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775764970.182360:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775764970.182371:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967075072/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775764970.182381:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775764970.182384:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775764970.182385:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775764970.182389:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1591 00000100:00080000:0.0:1775764970.182392:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775764970.182395:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775764970.182400:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775764970.182410:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967075200/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764970.182433:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967074944:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775764970.182448:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775764970.182452:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775764970.182453:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775764970.182458:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1591 00000100:00080000:0.0:1775764970.182462:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775764970.182466:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775764970.182471:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775764970.182481:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967075328/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764970.182537:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967075072:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775764970.182577:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967075200:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775764970.182602:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967075328:192.168.202.146@tcp:8:kworker.0 00010000:00010000:3.0:1775764971.077946:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764972.101891:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764973.125897:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775764973.254046:0:2366:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.202.146:988 00000400:00000100:3.0:1775764973.254093:0:2366:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775764973.254111:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:3.0:1775764973.254148:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:3.0:1775764973.254159:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:3.0:1775764973.254167:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:1.0:1775764973.254192:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764970/real 1775764973] req@ffff8f9970a75500 x1862022967075200/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775764995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764973.254220:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967075200/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775764995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764973.254230:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764973.254235:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775764973.254261:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967075200:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775764973.254268:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764970/real 1775764973] req@ffff8f9970a75180 x1862022967075328/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775764995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764973.254279:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967075328/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775764995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764973.254286:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764973.254288:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775764973.254291:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967075328:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775764973.254297:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764970/real 1775764973] req@ffff8f9970a77480 x1862022967074944/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764973.254308:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967074944/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764973.254315:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764973.254319:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775764973.254321:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967074944:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775764973.254327:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764970/real 1775764973] req@ffff8f9970a77800 x1862022967075072/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764973.254337:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967075072/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764973.254343:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764973.254346:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775764973.254362:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967075072:192.168.202.146@tcp:38:kworker.0 00010000:00010000:1.0:1775764974.149905:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764975.173876:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775764975.302436:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764975.302447:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775764975.302452:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764975.302462:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1596 00000100:00080000:3.0:1775764975.302473:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764975.302481:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775764975.302524:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967075456/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764975.302555:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764975.302559:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775764975.302560:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764975.302565:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1596 00000100:00080000:3.0:1775764975.302569:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764975.302611:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775764975.302620:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764975.302633:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967075584/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764975.302656:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764975.302659:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775764975.302661:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764975.302664:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1596 00000100:00080000:3.0:1775764975.302668:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764975.302676:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775764975.302681:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764975.302692:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967075712/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764975.302701:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764975.302704:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775764975.302706:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764975.302709:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1596 00000100:00080000:3.0:1775764975.302713:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764975.302717:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775764975.302722:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764975.302790:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967075840/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764975.302861:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967075456:192.168.202.146@tcp:250:kworker.0 00000800:00000100:1.0:1775764975.302921:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775764975.302960:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764975/real 1775764975] req@ffff8f9970edea00 x1862022967075456/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764975.302976:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967075456/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764975.302986:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764975.302990:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775764975.302993:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967075456:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775764975.303001:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967075584:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775764975.303024:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775764975.303036:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764975/real 1775764975] req@ffff8f9970edc380 x1862022967075584/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764975.303046:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967075584/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764975.303054:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764975.303056:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775764975.303059:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967075584:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775764975.303064:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967075712:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775764975.303107:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775764975.303120:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764975/real 1775764975] req@ffff8f9970edfb80 x1862022967075712/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764975.303131:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967075712/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764975.303137:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764975.303140:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775764975.303143:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967075712:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775764975.303148:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967075840:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775764975.303165:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775764975.303174:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764975/real 1775764975] req@ffff8f9970edd880 x1862022967075840/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764975.303184:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967075840/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764975.303192:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764975.303195:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775764975.303198:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967075840:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775764976.197948:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764977.221881:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764978.245884:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764979.269898:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764980.293975:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775764980.421979:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764980.421991:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775764980.421996:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764980.422007:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1602 00000100:00080000:3.0:1775764980.422019:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764980.422027:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775764980.422066:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967075968/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764980.422108:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764980.422113:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775764980.422116:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764980.422122:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1602 00000100:00080000:3.0:1775764980.422126:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764980.422130:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775764980.422139:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764980.422150:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967076096/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764980.422159:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764980.422162:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775764980.422164:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764980.422167:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1602 00000100:00080000:3.0:1775764980.422171:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764980.422174:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775764980.422178:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764980.422219:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967076224/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764980.422227:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764980.422230:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775764980.422231:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764980.422234:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1602 00000100:00080000:3.0:1775764980.422237:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764980.422240:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775764980.422243:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764980.422251:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967076352/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764980.422451:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967075968:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775764980.422543:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967076096:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775764980.422583:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967076224:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775764980.422607:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967076352:192.168.202.146@tcp:8:kworker.0 00000400:00000100:3.0:1775764980.934009:0:2368:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.202.146:988 00000400:00000100:3.0:1775764980.934060:0:2368:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775764980.934073:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:3.0:1775764980.934097:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:3.0:1775764980.934103:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:3.0:1775764980.934111:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:2.0:1775764980.934132:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764980/real 1775764980] req@ffff8f9970edc700 x1862022967075968/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764980.934151:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967075968/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764980.934159:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764980.934176:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775764980.934179:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967075968:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775764980.934184:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764980/real 1775764980] req@ffff8f9970edf800 x1862022967076096/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764980.934191:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967076096/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764980.934196:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764980.934198:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775764980.934200:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967076096:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775764980.934204:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764980/real 1775764980] req@ffff8f9970ede680 x1862022967076224/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764980.934210:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967076224/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764980.934213:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764980.934215:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775764980.934217:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967076224:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775764980.934220:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764980/real 1775764980] req@ffff8f9970ede300 x1862022967076352/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764980.934226:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967076352/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764980.934229:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764980.934238:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775764980.934240:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967076352:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775764981.317953:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764982.341884:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764983.365867:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764984.389926:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764985.413916:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775764985.541959:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764985.541995:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775764985.542000:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775764985.542013:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1607 00000100:00080000:1.0:1775764985.542027:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775764985.542035:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775764985.542073:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967076480/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775764985.542098:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764985.542102:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775764985.542104:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775764985.542108:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1607 00000100:00080000:1.0:1775764985.542111:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775764985.542115:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775764985.542124:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775764985.542135:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967076608/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775764985.542144:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764985.542147:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775764985.542149:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775764985.542152:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1607 00000100:00080000:1.0:1775764985.542155:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775764985.542158:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775764985.542162:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775764985.542168:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967076736/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775764985.542197:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764985.542200:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775764985.542202:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775764985.542205:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1607 00000100:00080000:1.0:1775764985.542208:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775764985.542211:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775764985.542215:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775764985.542225:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967076864/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764985.542769:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967076480:192.168.202.146@tcp:250:kworker.0 00000800:00000100:2.0:1775764985.542846:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775764985.542890:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764985/real 1775764985] req@ffff8f994f2e7b80 x1862022967076480/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764985.542908:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967076480/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764985.542917:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764985.542921:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775764985.542925:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967076480:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775764985.542931:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967076608:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775764985.542959:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775764985.542971:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764985/real 1775764985] req@ffff8f994f2e4700 x1862022967076608/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764985.542982:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967076608/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764985.542990:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764985.542993:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775764985.542996:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967076608:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775764985.543019:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967076736:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775764985.543039:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775764985.543049:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764985/real 1775764985] req@ffff8f994f2e6680 x1862022967076736/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764985.543060:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967076736/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764985.543067:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764985.543070:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775764985.543074:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967076736:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775764985.543079:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967076864:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775764985.543096:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775764985.543105:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764985/real 1775764985] req@ffff8f994f2e5f80 x1862022967076864/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764985.543115:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967076864/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764985.543122:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764985.543125:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775764985.543128:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967076864:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775764986.437944:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764987.461876:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764988.485862:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764989.509883:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764990.533906:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775764990.661934:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764990.661946:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775764990.661951:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775764990.661961:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1612 00000100:00080000:1.0:1775764990.661984:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775764990.661993:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775764990.662042:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967076992/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775764990.662081:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764990.662084:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775764990.662086:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775764990.662090:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1612 00000100:00080000:1.0:1775764990.662134:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775764990.662138:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775764990.662148:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775764990.662161:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967077120/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775764990.662171:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764990.662174:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775764990.662176:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775764990.662179:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1612 00000100:00080000:1.0:1775764990.662183:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775764990.662187:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775764990.662192:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775764990.662206:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967076992:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775764990.662207:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967077248/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775764990.662221:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764990.662224:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775764990.662226:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775764990.662229:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1612 00000100:00080000:1.0:1775764990.662232:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775764990.662239:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775764990.662243:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775764990.662255:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967077376/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764990.662358:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967077120:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775764990.662401:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967077248:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775764990.662427:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967077376:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775764991.557916:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764992.581876:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764993.605923:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775764993.734613:0:2367:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.202.146:988 00000400:00000100:2.0:1775764993.734713:0:2367:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775764993.734787:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:2.0:1775764993.734864:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:2.0:1775764993.734878:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:2.0:1775764993.734894:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:1.0:1775764993.734933:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764990/real 1775764993] req@ffff8f994f2e7800 x1862022967077248/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764993.734966:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967077248/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764993.734977:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764993.734982:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775764993.734987:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967077248:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775764993.735000:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764990/real 1775764993] req@ffff8f994f2e5180 x1862022967077376/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764993.735013:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967077376/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764993.735021:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764993.735024:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775764993.735028:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967077376:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775764993.735035:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764990/real 1775764993] req@ffff8f994f2e4a80 x1862022967076992/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764993.735047:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967076992/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764993.735054:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764993.735102:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775764993.735106:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967076992:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775764993.735113:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764990/real 1775764993] req@ffff8f994f2e6a00 x1862022967077120/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764993.735124:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967077120/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764993.735135:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764993.735139:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775764993.735142:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967077120:192.168.202.146@tcp:38:kworker.0 00010000:00010000:2.0:1775764994.629887:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764995.653856:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775764995.781950:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764995.781964:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775764995.781968:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775764995.781995:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1617 00000100:00080000:2.0:1775764995.782009:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775764995.782016:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775764995.782054:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0000 x1862022967077504/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775764995.782085:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764995.782089:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775764995.782091:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775764995.782095:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1617 00000100:00080000:2.0:1775764995.782128:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775764995.782133:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775764995.782142:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775764995.782155:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0380 x1862022967077632/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764995.782156:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967077504:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775764995.782164:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764995.782167:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775764995.782169:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775764995.782172:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1617 00000100:00080000:2.0:1775764995.782175:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775764995.782178:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775764995.782182:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775764995.782191:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2680 x1862022967077760/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775764995.782199:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764995.782202:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775764995.782214:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775764995.782218:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1617 00000100:00080000:2.0:1775764995.782220:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775764995.782223:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:1.0:1775764995.782227:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:2.0:1775764995.782227:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775764995.782236:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3800 x1862022967077888/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764995.782264:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764995/real 1775764995] req@ffff8f99703d0000 x1862022967077504/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764995.782281:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0000 x1862022967077504/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764995.782291:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764995.782295:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775764995.782299:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967077504:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775764995.782321:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967077632:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775764995.782342:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775764995.782359:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764995/real 1775764995] req@ffff8f99703d0380 x1862022967077632/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764995.782393:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0380 x1862022967077632/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764995.782401:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764995.782403:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775764995.782407:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967077632:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775764995.782413:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967077760:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775764995.782432:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775764995.782443:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764995/real 1775764995] req@ffff8f99703d2680 x1862022967077760/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764995.782454:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2680 x1862022967077760/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764995.782461:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764995.782464:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775764995.782468:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967077760:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775764995.782473:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967077888:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775764995.782490:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775764995.782499:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764995/real 1775764995] req@ffff8f99703d3800 x1862022967077888/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764995.782511:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3800 x1862022967077888/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764995.783183:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764995.783188:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775764995.783192:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967077888:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775764996.677871:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764997.701914:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764998.725925:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764999.749939:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765000.773890:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765000.901911:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765000.901928:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765000.901934:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765000.901949:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1622 00000100:00080000:2.0:1775765000.901965:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765000.901973:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765000.902026:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1500 x1862022967078016/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765000.902068:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765000.902071:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765000.902074:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765000.902078:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1622 00000100:00080000:2.0:1775765000.902083:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765000.902087:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765000.902099:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765000.902113:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0e00 x1862022967078144/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765000.902122:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765000.902125:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765000.902127:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765000.902130:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1622 00000100:00100000:3.0:1775765000.902240:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967078016:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775765000.902242:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765000.902257:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765000.902268:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765000.902296:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0700 x1862022967078272/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765000.902314:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765000.902320:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765000.902323:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765000.902330:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1622 00000800:00000100:3.0:1775765000.902330:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775765000.902337:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765000.902342:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765000.902347:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765000.902360:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2d80 x1862022967078400/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765000.902373:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765000/real 1775765000] req@ffff8f99703d1500 x1862022967078016/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765000.902420:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1500 x1862022967078016/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765000.902430:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765000.902434:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765000.902438:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967078016:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765000.902446:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967078144:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775765000.902465:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765000.902477:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765000/real 1775765000] req@ffff8f99703d0e00 x1862022967078144/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765000.902500:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0e00 x1862022967078144/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765000.902507:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765000.902511:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765000.902513:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967078144:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775765000.902541:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967078272:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775765000.902561:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765000.902571:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765000/real 1775765000] req@ffff8f99703d0700 x1862022967078272/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765000.902582:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0700 x1862022967078272/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765000.902595:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765000.902599:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765000.902601:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967078272:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775765000.902605:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967078400:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775765000.902619:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765000.902625:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765000/real 1775765000] req@ffff8f99703d2d80 x1862022967078400/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765000.902632:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2d80 x1862022967078400/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765000.902637:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765000.902639:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765000.902640:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967078400:192.168.202.146@tcp:8:kworker.0 00010000:00010000:3.0:1775765001.797814:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765002.821903:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765003.845903:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765004.869932:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765005.893902:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765006.022851:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765006.022860:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765006.022866:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765006.022877:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1627 00000100:00080000:0.0:1775765006.022889:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765006.022902:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765006.022939:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967078528/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765006.022967:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765006.022970:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765006.022972:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765006.022976:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1627 00000100:00080000:0.0:1775765006.022980:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765006.022985:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765006.023111:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765006.023141:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967078656/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765006.023155:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765006.023158:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765006.023161:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775765006.023166:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967078528:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775765006.023167:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1627 00000100:00080000:0.0:1775765006.023176:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765006.023182:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765006.023186:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765006.023196:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967078784/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765006.023204:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765006.023206:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765006.023207:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765006.023210:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1627 00000100:00080000:0.0:1775765006.023213:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765006.023215:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765006.023219:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765006.023226:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967078912/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775765006.023285:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765006.023325:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765006/real 1775765006] req@ffff8f9970a76a00 x1862022967078528/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765006.023345:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967078528/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765006.023356:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765006.023360:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765006.023365:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967078528:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765006.023393:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967078656:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775765006.023425:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765006.023438:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765006/real 1775765006] req@ffff8f9970a76d80 x1862022967078656/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765006.023458:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967078656/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765006.023464:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765006.023466:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765006.023468:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967078656:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775765006.023473:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967078784:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775765006.023488:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765006.023494:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765006/real 1775765006] req@ffff8f9970a74700 x1862022967078784/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765006.023501:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967078784/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765006.023506:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765006.023508:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765006.023509:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967078784:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775765006.023513:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967078912:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775765006.023523:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765006.023529:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765006/real 1775765006] req@ffff8f9970a75c00 x1862022967078912/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765006.023536:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967078912/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765006.023540:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765006.023542:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765006.023543:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967078912:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765006.917875:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765007.941887:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765008.965891:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765009.989932:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765011.013885:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765011.142257:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765011.142268:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765011.142273:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765011.142293:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1632 00000100:00080000:2.0:1775765011.142304:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765011.142312:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765011.142350:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3100 x1862022967079040/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765011.142433:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765011.142438:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765011.142440:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765011.142445:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1632 00000100:00080000:2.0:1775765011.142449:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765011.142453:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765011.142463:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765011.142478:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0a80 x1862022967079168/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765011.142490:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765011.142496:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1775765011.142496:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967079040:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775765011.142499:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765011.142507:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1632 00000100:00080000:2.0:1775765011.142512:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765011.142516:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765011.142522:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765011.142535:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1f80 x1862022967079296/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765011.142546:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765011.142550:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765011.142552:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765011.142556:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1632 00000100:00080000:2.0:1775765011.142573:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765011.142576:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765011.142580:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765011.142593:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1180 x1862022967079424/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765011.142641:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967079168:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775765011.142678:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967079296:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775765011.142702:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967079424:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765012.037927:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765013.061994:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765014.085881:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1775765014.214227:0:2369:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:0.0:1775765014.214274:0:2369:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1775765014.214335:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775765014.214385:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775765014.214396:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775765014.214409:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:3.0:1775765014.214543:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765011/real 1775765014] req@ffff8f99703d0a80 x1862022967079168/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765014.214569:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0a80 x1862022967079168/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765014.214584:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765014.214589:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765014.214593:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967079168:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775765014.214601:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765011/real 1775765014] req@ffff8f99703d1f80 x1862022967079296/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765014.214615:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1f80 x1862022967079296/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765014.214623:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765014.214626:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765014.214629:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967079296:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775765014.214635:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765011/real 1775765014] req@ffff8f99703d1180 x1862022967079424/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765014.214646:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1180 x1862022967079424/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765014.214653:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765014.214656:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765014.214659:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967079424:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775765014.214665:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765011/real 1775765014] req@ffff8f99703d3100 x1862022967079040/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765014.214678:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3100 x1862022967079040/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765014.214686:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765014.214690:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765014.214695:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967079040:192.168.202.146@tcp:250:kworker.0 00010000:00010000:1.0:1775765015.109907:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765016.133927:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765016.262204:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765016.262220:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765016.262225:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765016.262241:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1637 00000100:00080000:3.0:1775765016.262260:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765016.262271:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765016.262330:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967079552/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765016.262383:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765016.262388:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765016.262390:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765016.262395:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1637 00000100:00080000:3.0:1775765016.262400:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765016.262404:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765016.262417:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765016.262432:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967079680/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765016.262442:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765016.262445:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765016.262447:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765016.262451:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1637 00000100:00080000:3.0:1775765016.262455:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765016.262459:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765016.262465:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765016.262477:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967079808/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765016.262666:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765016.262671:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765016.262674:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765016.262683:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1637 00000100:00080000:3.0:1775765016.262694:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765016.262701:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765016.262714:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765016.262792:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967079936/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765016.263280:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967079552:192.168.202.146@tcp:250:kworker.0 00000800:00000100:0.0:1775765016.263396:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765016.263450:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765016/real 1775765016] req@ffff8f9970eded80 x1862022967079552/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765016.263471:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967079552/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765016.263479:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765016.263483:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765016.263486:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967079552:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765016.263493:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967079680:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775765016.263516:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765016.263529:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765016/real 1775765016] req@ffff8f9970edc000 x1862022967079680/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765016.263541:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967079680/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765016.263549:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765016.263553:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765016.263556:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967079680:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775765016.263797:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967079808:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765016.263874:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765016.263900:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765016/real 1775765016] req@ffff8f9970edce00 x1862022967079808/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765016.263918:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967079808/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765016.263928:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765016.263932:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765016.263936:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967079808:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765016.263943:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967079936:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765016.263973:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765016.263984:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765016/real 1775765016] req@ffff8f9970edd500 x1862022967079936/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765016.263996:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967079936/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765016.264003:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765016.264006:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765016.264009:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967079936:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775765017.157904:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765018.181915:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765019.205972:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765020.229848:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765021.253930:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765021.381858:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765021.381874:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765021.381879:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765021.381889:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1642 00000100:00080000:3.0:1775765021.381901:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765021.381908:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765021.381948:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf100 x1862022967080064/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765021.381977:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765021.381981:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765021.381982:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765021.381987:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1642 00000100:00080000:3.0:1775765021.381991:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765021.382026:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765021.382037:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765021.382051:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddc00 x1862022967080192/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765021.382061:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765021.382064:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765021.382066:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765021.382070:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1642 00000100:00080000:3.0:1775765021.382073:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765021.382077:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765021.382082:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765021.382095:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967080320/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765021.382104:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765021.382107:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765021.382109:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765021.382112:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1642 00000100:00080000:3.0:1775765021.382116:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765021.382119:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765021.382124:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765021.382135:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967080448/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765021.383235:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967080064:192.168.202.146@tcp:250:kworker.0 00000800:00000100:0.0:1775765021.383323:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765021.383364:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765021/real 1775765021] req@ffff8f9970edf100 x1862022967080064/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765021.383413:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf100 x1862022967080064/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765021.383422:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765021.383427:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765021.383431:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967080064:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765021.383439:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967080192:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775765021.383465:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765021.383478:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765021/real 1775765021] req@ffff8f9970eddc00 x1862022967080192/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765021.383492:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddc00 x1862022967080192/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765021.383500:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765021.383504:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765021.383507:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967080192:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775765021.383514:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967080320:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765021.383537:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765021.383549:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765021/real 1775765021] req@ffff8f9970edd500 x1862022967080320/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765021.383562:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967080320/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765021.383594:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765021.383598:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765021.383600:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967080320:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765021.383606:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967080448:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765021.383639:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765021.383650:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765021/real 1775765021] req@ffff8f9970edce00 x1862022967080448/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765021.383663:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967080448/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765021.383672:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765021.383676:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765021.383679:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967080448:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775765022.277866:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765023.301917:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765024.325873:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765025.349882:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765026.373823:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765026.501908:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765026.501929:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765026.501935:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765026.501953:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1648 00000100:00080000:3.0:1775765026.501968:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765026.501977:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765026.502030:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967080576/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765026.502070:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765026.502075:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765026.502078:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765026.502084:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1648 00000100:00080000:3.0:1775765026.502088:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765026.502093:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765026.502105:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765026.502120:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967080704/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765026.502131:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765026.502135:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765026.502137:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765026.502141:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1648 00000100:00080000:3.0:1775765026.502145:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765026.502149:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765026.502155:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765026.502190:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967080576:192.168.202.146@tcp:250:kworker.0 00000800:00000100:0.0:1775765026.502300:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765026.502354:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765026/real 1775765026] req@ffff8f9970edc000 x1862022967080576/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765026.502381:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967080576/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765026.502396:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765026.502402:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765026.502409:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967080576:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765026.502429:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967080832/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765026.502454:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765026.502458:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765026.502461:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765026.502466:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1648 00000100:00080000:3.0:1775765026.502478:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765026.502485:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765026.502495:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765026.502507:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967080960/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765026.502537:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967080704:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775765026.502613:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765026.502651:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765026/real 1775765026] req@ffff8f9970eded80 x1862022967080704/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765026.502670:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967080704/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765026.502678:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765026.502681:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765026.502684:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967080704:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775765026.502708:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967080832:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765026.502725:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765026.502732:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765026/real 1775765026] req@ffff8f9970ede300 x1862022967080832/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765026.502789:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967080832/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765026.502795:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765026.502797:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765026.502800:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967080832:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765026.502804:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967080960:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765026.502817:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765026.502824:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765026/real 1775765026] req@ffff8f9970ede680 x1862022967080960/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765026.502857:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967080960/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765026.502862:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765026.502864:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765026.502866:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967080960:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775765027.397952:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765028.421851:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765029.445915:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765030.469875:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765031.493880:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765031.621855:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765031.621868:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765031.621873:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765031.621895:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1653 00000100:00080000:3.0:1775765031.621907:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765031.621916:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765031.621953:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967081088/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765031.621984:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765031.621988:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765031.621991:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765031.621996:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1653 00000100:00080000:3.0:1775765031.622000:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765031.622004:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765031.622014:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765031.622028:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967081216/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765031.622038:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967081088:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775765031.622042:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765031.622049:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765031.622051:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765031.622055:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1653 00000100:00080000:3.0:1775765031.622059:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765031.622064:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765031.622070:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765031.622083:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967081344/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765031.622108:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765031.622111:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765031.622112:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765031.622117:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1653 00000100:00080000:3.0:1775765031.622122:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765031.622126:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765031.622131:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765031.622142:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967081472/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765031.622150:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765031.622230:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765031/real 1775765031] req@ffff8f9970edf800 x1862022967081088/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765031.622254:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967081088/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765031.622265:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765031.622271:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765031.622275:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967081088:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765031.622300:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967081216:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775765031.622325:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765031.622338:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765031/real 1775765031] req@ffff8f9970edc700 x1862022967081216/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765031.622351:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967081216/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765031.622400:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765031.622405:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765031.622409:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967081216:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775765031.622417:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967081344:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765031.622439:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765031.622450:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765031/real 1775765031] req@ffff8f9970edd880 x1862022967081344/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765031.622465:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967081344/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765031.622473:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765031.622476:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765031.622480:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967081344:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765031.622485:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967081472:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765031.622506:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765031.622518:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765031/real 1775765031] req@ffff8f9970edfb80 x1862022967081472/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765031.622529:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967081472/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765031.622535:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765031.622537:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765031.622553:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967081472:192.168.202.146@tcp:8:kworker.0 00010000:00010000:3.0:1775765032.517910:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765033.541905:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765034.565941:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765035.590019:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765036.613932:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765036.742265:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765036.742282:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765036.742288:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765036.742305:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1658 00000100:00080000:0.0:1775765036.742322:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765036.742333:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765036.742388:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967081600/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765036.742426:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765036.742436:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765036.742438:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765036.742444:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1658 00000100:00080000:0.0:1775765036.742450:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765036.742456:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765036.742469:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765036.742473:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967081600:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765036.742490:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967081728/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765036.742504:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765036.742509:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765036.742511:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765036.742516:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1658 00000100:00080000:0.0:1775765036.742522:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1775765036.742576:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765036.742641:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765036/real 1775765036] req@ffff8f9970a74380 x1862022967081600/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765036.742666:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967081600/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765036.742680:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765036.742686:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765036.742692:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967081600:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765036.742723:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967081728:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775765036.742813:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765036.742838:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765036/real 1775765036] req@ffff8f9970a75c00 x1862022967081728/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765036.742848:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765036.742858:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967081728/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1775765036.742870:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1775765036.742871:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765036.742882:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765036.742886:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967081728:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775765036.742925:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967081856/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765036.742967:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765036.742974:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765036.742978:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765036.742986:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1658 00000100:00100000:3.0:1775765036.742995:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967081856:192.168.202.146@tcp:8:kworker.0 00000100:00080000:0.0:1775765036.742999:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765036.743009:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765036.743018:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775765036.743035:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765036.743041:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967081984/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765036.743055:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765036/real 1775765036] req@ffff8f9970a74700 x1862022967081856/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765036.743231:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967081856/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765036.743245:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765036.743250:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765036.743256:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967081856:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775765036.743293:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967081984:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775765036.743349:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765036.743381:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765036/real 1775765036] req@ffff8f9970a76d80 x1862022967081984/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765036.743392:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967081984/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765036.743399:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765036.743402:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765036.743405:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967081984:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765037.637931:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765038.661830:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765039.685879:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765040.709892:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765041.733866:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765041.861906:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765041.861953:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765041.861958:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765041.861978:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1663 00000100:00080000:1.0:1775765041.861994:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765041.862002:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765041.862052:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967082112/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765041.862085:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765041.862090:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765041.862093:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765041.862098:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1663 00000100:00080000:1.0:1775765041.862103:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765041.862124:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765041.862134:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765041.862147:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967082112:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775765041.862154:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967082240/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765041.862169:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765041.862174:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765041.862176:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765041.862182:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1663 00000100:00080000:1.0:1775765041.862188:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765041.862193:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765041.862200:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765041.862214:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967082368/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765041.862215:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765041.862247:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765041/real 1775765041] req@ffff8f994f2e6a00 x1862022967082112/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765041.862248:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765041.862258:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765041.862261:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765041.862268:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1663 00000100:00100000:2.0:1775765041.862273:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967082112/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765041.862275:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765041.862284:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00080000:2.0:1775765041.862287:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1775765041.862291:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1775765041.862293:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765041.862298:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967082112:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775765041.862310:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967082496/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765041.862326:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967082240:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765041.862374:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765041.862389:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765041/real 1775765041] req@ffff8f994f2e4a80 x1862022967082240/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765041.862400:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967082240/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765041.862407:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765041.862411:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765041.862413:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967082240:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765041.862421:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967082368:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765041.862437:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765041.862446:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765041/real 1775765041] req@ffff8f994f2e5180 x1862022967082368/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765041.862458:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967082368/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765041.862464:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765041.862468:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765041.862472:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967082368:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765041.862484:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967082496:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765041.862497:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765041.862505:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765041/real 1775765041] req@ffff8f994f2e7800 x1862022967082496/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765041.862517:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967082496/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765041.862522:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765041.862524:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765041.862526:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967082496:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765042.757855:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765043.781900:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765044.805858:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765045.829916:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765046.853924:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765046.982227:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765046.982242:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765046.982248:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765046.982263:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1668 00000100:00080000:3.0:1775765046.982278:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765046.982287:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765046.982448:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967082624/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765046.982474:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765046.982478:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765046.982480:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765046.982485:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1668 00000100:00080000:3.0:1775765046.982493:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765046.982499:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765046.982507:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765046.982518:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967082624:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765046.982518:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967082752/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765046.982532:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765046.982535:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765046.982536:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765046.982538:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1668 00000100:00080000:3.0:1775765046.982541:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765046.982544:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765046.982547:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765046.982557:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967082880/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765046.982563:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765046.982565:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765046.982566:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765046.982587:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1668 00000100:00080000:3.0:1775765046.982590:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765046.982592:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765046.982596:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765046.982605:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967083008/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765046.982634:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967082752:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765046.982659:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967082880:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765046.982682:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967083008:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765047.877900:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765048.901994:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765049.925928:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775765050.054166:0:2366:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:3.0:1775765050.054218:0:2366:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775765050.054239:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:3.0:1775765050.054282:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:3.0:1775765050.054292:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:3.0:1775765050.054301:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:0.0:1775765050.055125:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765046/real 1775765050] req@ffff8f9970edea00 x1862022967082752/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765050.055164:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967082752/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765050.055181:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765050.055187:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765050.055192:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967082752:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775765050.055199:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765046/real 1775765050] req@ffff8f9970edf480 x1862022967082880/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765050.055244:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967082880/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765050.055253:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765050.055258:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765050.055262:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967082880:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765050.055270:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765046/real 1775765050] req@ffff8f9970edca80 x1862022967083008/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765050.055283:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967083008/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765050.055288:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765050.055291:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765050.055295:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967083008:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765050.055303:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765046/real 1775765050] req@ffff8f9970edc380 x1862022967082624/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765050.055316:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967082624/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765050.055324:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765050.055328:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765050.055332:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967082624:192.168.202.146@tcp:250:kworker.0 00010000:00010000:3.0:1775765050.949842:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765051.974138:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765052.101939:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765052.101963:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765052.101969:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765052.101982:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1673 00000100:00080000:1.0:1775765052.101998:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765052.102006:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765052.102062:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967083136/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765052.102091:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765052.102096:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765052.102099:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775765052.102142:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967083136:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775765052.102152:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1673 00000100:00080000:1.0:1775765052.102158:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765052.102163:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765052.102175:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765052.102195:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967083264/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765052.102212:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765052.102216:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765052.102219:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765052.102223:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1673 00000100:00080000:1.0:1775765052.102227:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765052.102234:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765052.102241:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765052.102276:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967083392/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765052.102289:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765052.102292:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765052.102294:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765052.102298:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1673 00000100:00080000:1.0:1775765052.102302:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765052.102307:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765052.102312:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765052.102325:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967083520/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765052.102429:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765052.102516:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765052/real 1775765052] req@ffff8f994f2e6d80 x1862022967083136/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765052.102551:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967083136/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765052.102564:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765052.102569:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765052.102573:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967083136:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765052.102606:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967083264:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775765052.102637:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765052.102649:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765052/real 1775765052] req@ffff8f994f2e4000 x1862022967083264/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765052.102670:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967083264/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765052.102677:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765052.102680:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765052.102682:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967083264:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775765052.102688:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967083392:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765052.102704:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765052.102712:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765052/real 1775765052] req@ffff8f994f2e7800 x1862022967083392/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765052.102791:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967083392/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765052.102801:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765052.102804:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765052.102807:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967083392:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765052.102815:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967083520:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765052.102840:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765052.102853:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765052/real 1775765052] req@ffff8f994f2e5180 x1862022967083520/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765052.102868:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967083520/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765052.102877:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765052.102881:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765052.102886:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967083520:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775765052.997881:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765054.021887:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765055.046012:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765056.069947:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765057.093863:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765057.222090:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765057.222101:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765057.222106:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765057.222117:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1678 00000100:00080000:3.0:1775765057.222129:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765057.222142:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765057.222179:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd180 x1862022967083648/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765057.222221:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765057.222225:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765057.222228:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765057.222274:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1678 00000100:00100000:2.0:1775765057.222278:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967083648:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775765057.222279:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765057.222283:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765057.222291:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765057.222303:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddf80 x1862022967083776/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765057.222314:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765057.222317:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765057.222319:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765057.222322:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1678 00000100:00080000:3.0:1775765057.222326:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765057.222330:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765057.222334:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765057.222340:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765057.222343:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967083904/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765057.222352:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765057.222355:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765057.222356:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765057.222360:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1678 00000100:00080000:3.0:1775765057.222363:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765057.222366:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765057.222370:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765057.222380:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967084032/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765057.222386:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765057/real 1775765057] req@ffff8f9970edd180 x1862022967083648/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765057.222406:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd180 x1862022967083648/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765057.222419:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765057.222423:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765057.222428:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967083648:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765057.222453:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967083776:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765057.222474:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765057.222484:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765057/real 1775765057] req@ffff8f9970eddf80 x1862022967083776/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765057.222492:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddf80 x1862022967083776/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765057.222497:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765057.222500:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765057.222502:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967083776:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765057.222506:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967083904:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765057.222517:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765057.222523:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765057/real 1775765057] req@ffff8f9970edc380 x1862022967083904/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765057.222530:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967083904/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765057.222535:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765057.222537:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765057.222539:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967083904:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765057.222558:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967084032:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765057.222571:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765057.222578:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765057/real 1775765057] req@ffff8f9970edca80 x1862022967084032/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765057.222585:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967084032/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765057.222590:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765057.222592:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765057.222594:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967084032:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775765058.117918:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765059.141908:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765060.165918:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765061.189876:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765062.213881:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765062.342243:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765062.342255:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765062.342260:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765062.342272:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1683 00000100:00080000:1.0:1775765062.342286:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765062.342295:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765062.342351:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967084160/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765062.342387:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765062.342391:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765062.342393:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765062.342397:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1683 00000100:00080000:1.0:1775765062.342401:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765062.342406:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765062.342418:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765062.342435:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967084160:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775765062.342436:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967084288/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765062.342466:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765062.342469:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765062.342470:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1775765062.342528:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765062.342566:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765062/real 1775765062] req@ffff8f994f2e4a80 x1862022967084160/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765062.342578:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1683 00000100:00080000:1.0:1775765062.342586:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775765062.342590:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967084160/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765062.342595:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00080000:2.0:1775765062.342607:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1775765062.342612:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1775765062.342612:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765062.342618:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967084160:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765062.342641:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967084288:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775765062.342645:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967084416/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765062.342663:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765062.342670:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765062.342674:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765062.342679:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1683 00000100:00080000:1.0:1775765062.342685:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775765062.342690:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775765062.342691:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765062.342700:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765062.342705:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765062/real 1775765062] req@ffff8f994f2e6a00 x1862022967084288/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765062.342719:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967084288/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765062.342726:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765062.342731:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765062.342732:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967084544/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765062.342734:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967084288:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765062.342767:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967084416:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765062.342784:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765062.342795:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765062/real 1775765062] req@ffff8f994f2e5f80 x1862022967084416/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765062.342804:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967084416/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765062.342810:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765062.342815:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765062.342819:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967084416:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765062.342832:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967084544:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765062.342851:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765062.342874:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765062/real 1775765062] req@ffff8f994f2e6680 x1862022967084544/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765062.342916:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967084544/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765062.342938:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765062.342941:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765062.342944:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967084544:192.168.202.146@tcp:8:kworker.0 00010000:00010000:3.0:1775765063.237837:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765064.261910:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765065.285875:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765066.309925:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765067.333911:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765067.461855:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765067.461866:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765067.461882:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765067.461895:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1689 00000100:00080000:1.0:1775765067.461909:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765067.461916:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765067.461973:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967084672/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765067.462135:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765067.462138:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765067.462141:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765067.462149:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1689 00000100:00080000:1.0:1775765067.462159:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765067.462166:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765067.462176:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967084672:192.168.202.146@tcp:250:kworker.0 00000008:00100000:1.0:1775765067.462177:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765067.462214:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967084800/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765067.462223:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765067.462225:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765067.462227:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765067.462230:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1689 00000100:00080000:1.0:1775765067.462233:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765067.462235:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765067.462238:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765067.462246:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967084928/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765067.462253:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765067.462256:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765067.462257:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1775765067.462262:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775765067.462263:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1689 00000100:00080000:1.0:1775765067.462279:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765067.462284:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765067.462288:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765067.462299:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967085056/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765067.462313:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765067/real 1775765067] req@ffff8f994f2e4700 x1862022967084672/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765067.462333:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967084672/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765067.462343:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765067.462348:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765067.462352:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967084672:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765067.462376:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967084800:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765067.462392:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765067.462401:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765067/real 1775765067] req@ffff8f994f2e7b80 x1862022967084800/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765067.462409:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967084800/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765067.462505:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765067.462509:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765067.462513:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967084800:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765067.462522:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967084928:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765067.462572:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765067.462593:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765067/real 1775765067] req@ffff8f994f2e6300 x1862022967084928/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765067.462603:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967084928/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765067.462609:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765067.462611:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765067.462613:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967084928:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765067.462616:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967085056:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765067.462627:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765067.462633:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765067/real 1775765067] req@ffff8f994f2e4e00 x1862022967085056/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765067.462640:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967085056/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765067.462648:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765067.462651:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765067.462669:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967085056:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765068.357955:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765069.381901:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765070.405918:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765071.429915:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765072.453898:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765072.582217:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765072.582228:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765072.582232:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765072.582242:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1694 00000100:00080000:2.0:1775765072.582267:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765072.582275:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765072.582309:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3480 x1862022967085184/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765072.582333:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765072.582337:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765072.582339:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765072.582344:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1694 00000100:00080000:2.0:1775765072.582349:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765072.582353:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765072.582362:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765072.582375:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967085184:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765072.582375:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2300 x1862022967085312/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765072.582385:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765072.582389:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765072.582390:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765072.582394:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1694 00000100:00080000:2.0:1775765072.582434:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765072.582438:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:0.0:1775765072.582440:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:2.0:1775765072.582443:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765072.582454:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1c00 x1862022967085440/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765072.582463:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765072.582471:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765072.582473:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765072.582477:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1694 00000100:00080000:2.0:1775765072.582481:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765072.582484:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765072.582489:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765072.582493:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765072/real 1775765072] req@ffff8f99703d3480 x1862022967085184/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765072.582498:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3b80 x1862022967085568/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765072.582518:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3480 x1862022967085184/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765072.582553:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765072.582556:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765072.582559:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967085184:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765072.582579:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967085312:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775765072.582599:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765072.582607:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765072/real 1775765072] req@ffff8f99703d2300 x1862022967085312/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765072.582616:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2300 x1862022967085312/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765072.582621:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765072.582623:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765072.582625:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967085312:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775765072.582629:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967085440:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765072.582640:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765072.582645:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765072/real 1775765072] req@ffff8f99703d1c00 x1862022967085440/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765072.582653:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1c00 x1862022967085440/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765072.582658:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765072.582659:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765072.582661:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967085440:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765072.582678:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967085568:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765072.582689:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765072.582696:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765072/real 1775765072] req@ffff8f99703d3b80 x1862022967085568/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765072.582703:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3b80 x1862022967085568/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765072.582709:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765072.582712:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765072.582715:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967085568:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765073.477933:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765074.501873:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765075.525984:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765076.549963:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765077.573936:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765077.701858:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765077.701870:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765077.701875:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765077.701887:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1699 00000100:00080000:2.0:1775765077.701901:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765077.701910:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765077.701963:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1880 x1862022967085696/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765077.702120:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765077.702124:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765077.702127:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765077.702136:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1699 00000100:00080000:2.0:1775765077.702146:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765077.702155:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765077.702161:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967085696:192.168.202.146@tcp:250:kworker.0 00000008:00100000:2.0:1775765077.702170:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765077.702200:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2a00 x1862022967085824/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765077.702209:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765077.702212:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765077.702213:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765077.702217:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1699 00000100:00080000:2.0:1775765077.702220:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765077.702223:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:0.0:1775765077.702228:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:2.0:1775765077.702231:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765077.702243:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3b80 x1862022967085952/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765077.702250:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765077.702254:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765077.702257:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765077.702264:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1699 00000100:00100000:0.0:1775765077.702281:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765077/real 1775765077] req@ffff8f99703d1880 x1862022967085696/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765077.702285:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765077.702293:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765077.702298:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765077.702304:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1880 x1862022967085696/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765077.702311:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1c00 x1862022967086080/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765077.702314:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765077.702323:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765077.702328:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967085696:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765077.702349:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967085824:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775765077.702368:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765077.702380:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765077/real 1775765077] req@ffff8f99703d2a00 x1862022967085824/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765077.702408:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2a00 x1862022967085824/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765077.702415:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765077.702417:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765077.702420:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967085824:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775765077.702429:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967085952:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765077.702448:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765077.702457:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765077/real 1775765077] req@ffff8f99703d3b80 x1862022967085952/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765077.702471:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3b80 x1862022967085952/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765077.702476:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765077.702480:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765077.702483:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967085952:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765077.702493:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967086080:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765077.702520:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765077.702543:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765077/real 1775765077] req@ffff8f99703d1c00 x1862022967086080/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765077.702551:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1c00 x1862022967086080/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765077.702571:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765077.702574:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765077.702576:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967086080:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765078.597901:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765079.621907:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765080.645846:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765081.669953:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765082.693874:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765082.821835:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765082.821846:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765082.821851:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765082.821862:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1704 00000100:00080000:3.0:1775765082.821874:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765082.821880:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765082.821916:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967086208/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765082.821946:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765082.821955:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765082.821957:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765082.821972:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1704 00000100:00080000:3.0:1775765082.821977:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765082.821982:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765082.821991:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967086208:192.168.202.146@tcp:250:kworker.0 00000008:00100000:3.0:1775765082.821992:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765082.822011:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967086336/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765082.822024:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765082.822027:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:0.0:1775765082.822054:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775765082.822056:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765082.822063:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1704 00000100:00080000:3.0:1775765082.822070:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765082.822075:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765082.822081:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765082.822095:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967086464/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765082.822104:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765082/real 1775765082] req@ffff8f9970edf480 x1862022967086208/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765082.822108:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765082.822116:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765082.822120:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765082.822125:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1704 00000100:00100000:0.0:1775765082.822129:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967086208/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765082.822131:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765082.822140:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00080000:0.0:1775765082.822142:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765082.822147:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000008:00100000:3.0:1775765082.822147:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765082.822153:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967086208:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765082.822166:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967086592/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765082.822176:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967086336:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775765082.822194:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765082.822205:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765082/real 1775765082] req@ffff8f9970edea00 x1862022967086336/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765082.822213:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967086336/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765082.822219:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765082.822221:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765082.822223:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967086336:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775765082.822244:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967086464:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765082.822257:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765082.822264:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765082/real 1775765082] req@ffff8f9970edfb80 x1862022967086464/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765082.822272:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967086464/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765082.822277:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765082.822279:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765082.822281:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967086464:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765082.822290:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967086592:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765082.822300:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765082.822305:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765082/real 1775765082] req@ffff8f9970edd880 x1862022967086592/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765082.822311:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967086592/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765082.822316:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765082.822318:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765082.822320:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967086592:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775765083.717918:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765084.741900:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765085.765906:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765086.789921:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765087.813912:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765087.941857:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765087.941871:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765087.941878:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765087.941890:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1709 00000100:00080000:2.0:1775765087.941904:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765087.941911:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765087.941959:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2300 x1862022967086720/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765087.941989:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765087.941992:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765087.941993:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765087.941996:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1709 00000100:00080000:2.0:1775765087.941999:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765087.942001:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765087.942081:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967086720:192.168.202.146@tcp:250:kworker.0 00000008:00100000:2.0:1775765087.942127:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775765087.942155:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765087.942168:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3480 x1862022967086848/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765087.942191:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765087.942196:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765087.942200:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775765087.942207:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765087/real 1775765087] req@ffff8f99703d2300 x1862022967086720/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765087.942209:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1709 00000100:00080000:2.0:1775765087.942221:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765087.942227:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765087.942232:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765087.942242:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3100 x1862022967086976/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765087.942249:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765087.942251:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765087.942252:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765087.942254:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1709 00000100:00080000:2.0:1775765087.942256:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765087.942259:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765087.942261:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765087.942266:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1180 x1862022967087104/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765087.942335:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2300 x1862022967086720/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765087.942347:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765087.942353:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765087.942358:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967086720:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765087.942389:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967086848:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775765087.942443:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765087.942478:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765087/real 1775765087] req@ffff8f99703d3480 x1862022967086848/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765087.942494:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3480 x1862022967086848/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765087.942503:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765087.942507:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765087.942510:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967086848:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775765087.942517:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967086976:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765087.942540:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765087.942552:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765087/real 1775765087] req@ffff8f99703d3100 x1862022967086976/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765087.942566:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3100 x1862022967086976/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765087.942608:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765087.942612:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765087.942616:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967086976:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765087.942623:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967087104:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765087.942647:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765087.942660:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765087/real 1775765087] req@ffff8f99703d1180 x1862022967087104/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765087.942672:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1180 x1862022967087104/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765087.942711:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765087.942716:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765087.942720:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967087104:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775765088.837889:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765089.861886:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765090.885918:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765091.909890:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765092.933936:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765093.061845:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765093.061855:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765093.061860:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765093.061869:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1714 00000100:00080000:0.0:1775765093.061880:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765093.061894:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765093.061927:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967087232/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765093.061949:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765093.061970:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765093.061972:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765093.061975:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1714 00000100:00080000:0.0:1775765093.061979:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775765093.061982:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967087232:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775765093.061984:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765093.061996:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765093.062013:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967087360/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765093.062026:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765093.062029:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765093.062031:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765093.062035:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1714 00000100:00080000:0.0:1775765093.062040:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765093.062045:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765093.062051:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765093.062061:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765093.062066:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967087488/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765093.062077:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765093.062081:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765093.062084:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765093.062088:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1714 00000100:00080000:0.0:1775765093.062094:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765093.062098:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765093.062108:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765093/real 1775765093] req@ffff8f9970a76a00 x1862022967087232/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1775765093.062123:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765093.062127:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967087232/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765093.062137:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1775765093.062141:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967087616/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765093.062142:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765093.062149:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967087232:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765093.062175:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967087360:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765093.062202:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765093.062211:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765093/real 1775765093] req@ffff8f9970a77800 x1862022967087360/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765093.062220:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967087360/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765093.062225:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765093.062228:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765093.062230:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967087360:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765093.062234:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967087488:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765093.062248:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765093.062257:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765093/real 1775765093] req@ffff8f9970a77480 x1862022967087488/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765093.062301:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967087488/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765093.062311:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765093.062314:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765093.062318:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967087488:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765093.062324:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967087616:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765093.062341:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765093.062349:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765093/real 1775765093] req@ffff8f9970a75180 x1862022967087616/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765093.062357:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967087616/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765093.062362:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765093.062364:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765093.062366:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967087616:192.168.202.146@tcp:8:kworker.0 00010000:00010000:3.0:1775765093.957826:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765094.981961:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765096.005873:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765097.029946:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765098.053920:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765098.181884:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765098.181894:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765098.181898:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765098.181908:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1719 00000100:00080000:1.0:1775765098.181920:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765098.181927:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765098.181964:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7100 x1862022967087744/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765098.181996:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765098.181999:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765098.182001:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765098.182045:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1719 00000100:00080000:1.0:1775765098.182050:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775765098.182056:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967087744:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775765098.182058:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765098.182075:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765098.182094:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967087872/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765098.182106:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765098.182110:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765098.182111:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765098.182116:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1719 00000100:00080000:1.0:1775765098.182121:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765098.182124:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765098.182129:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765098.182146:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765098.182159:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967088000/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765098.182171:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765098.182175:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765098.182176:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765098.182182:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1719 00000100:00080000:1.0:1775765098.182188:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765098.182194:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765098.182202:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765098.182207:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765098/real 1775765098] req@ffff8f994f2e7100 x1862022967087744/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765098.182219:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967088128/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765098.182231:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7100 x1862022967087744/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765098.182412:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765098.182417:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765098.182421:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967087744:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765098.182446:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967087872:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765098.182498:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765098.182517:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765098/real 1775765098] req@ffff8f994f2e4e00 x1862022967087872/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765098.182528:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967087872/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765098.182533:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765098.182536:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765098.182538:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967087872:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765098.182541:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967088000:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765098.182552:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765098.182558:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765098/real 1775765098] req@ffff8f994f2e6300 x1862022967088000/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765098.182565:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967088000/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765098.182570:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765098.182571:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765098.182573:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967088000:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765098.182605:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967088128:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765098.182618:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765098.182625:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765098/real 1775765098] req@ffff8f994f2e7b80 x1862022967088128/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765098.182632:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967088128/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765098.182638:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765098.182640:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765098.182642:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967088128:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765099.077943:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765100.101815:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765101.125934:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765102.149884:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765103.173866:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765103.301848:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765103.301858:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765103.301862:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765103.301870:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1724 00000100:00080000:0.0:1775765103.301880:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765103.301886:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765103.301919:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967088256/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765103.301942:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765103.301945:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765103.301946:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765103.301950:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1724 00000100:00080000:0.0:1775765103.301952:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765103.301955:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765103.301963:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765103.301973:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967088384/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765103.301979:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967088256:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775765103.301985:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765103.301988:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765103.301989:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765103.301992:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1724 00000100:00080000:0.0:1775765103.302032:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765103.302035:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765103.302040:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765103.302041:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765103.302054:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967088512/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765103.302062:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765103.302065:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765103.302066:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765103.302070:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1724 00000100:00080000:0.0:1775765103.302073:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765103.302076:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765103.302079:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765103.302089:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765103/real 1775765103] req@ffff8f9970a75500 x1862022967088256/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765103.302090:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967088640/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765103.302134:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967088256/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765103.302147:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765103.302152:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765103.302157:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967088256:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765103.302183:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967088384:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765103.302250:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765103.302266:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765103/real 1775765103] req@ffff8f9970a76300 x1862022967088384/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765103.302282:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967088384/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765103.302292:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765103.302296:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765103.302300:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967088384:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765103.302307:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967088512:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765103.302334:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765103.302347:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765103/real 1775765103] req@ffff8f9970a77100 x1862022967088512/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765103.302362:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967088512/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765103.302371:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765103.302375:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765103.302378:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967088512:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765103.302386:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967088640:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765103.302410:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765103.302423:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765103/real 1775765103] req@ffff8f9970a75180 x1862022967088640/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765103.302468:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967088640/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765103.302477:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765103.302480:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765103.302484:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967088640:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765104.197921:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765105.221904:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765106.245855:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765107.270027:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765108.293937:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765108.421888:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765108.421897:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765108.421901:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765108.421911:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1730 00000100:00080000:0.0:1775765108.421923:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765108.421930:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765108.421963:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967088768/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765108.421994:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765108.422022:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765108.422024:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765108.422029:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1730 00000100:00080000:0.0:1775765108.422034:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765108.422041:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765108.422042:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967088768:192.168.202.146@tcp:250:kworker.0 00000008:00100000:0.0:1775765108.422050:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765108.422068:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967088896/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765108.422081:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765108.422085:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765108.422087:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765108.422092:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1730 00000100:00080000:0.0:1775765108.422096:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775765108.422100:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775765108.422104:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765108.422110:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765108.422123:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967089024/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765108.422133:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765108.422137:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765108.422138:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765108.422143:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1730 00000100:00100000:2.0:1775765108.422145:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765108/real 1775765108] req@ffff8f9970a77480 x1862022967088768/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765108.422148:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765108.422158:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765108.422174:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967088768/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1775765108.422182:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1775765108.422185:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765108.422190:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765108.422193:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967088768:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765108.422199:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967089152/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765108.422217:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967088896:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765108.422234:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765108.422244:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765108/real 1775765108] req@ffff8f9970a77800 x1862022967088896/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765108.422251:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967088896/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765108.422274:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765108.422277:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765108.422279:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967088896:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765108.422283:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967089024:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765108.422296:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765108.422303:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765108/real 1775765108] req@ffff8f9970a76a00 x1862022967089024/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765108.422314:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967089024/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765108.422322:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765108.422325:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765108.422329:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967089024:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765108.422335:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967089152:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765108.422355:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765108.422367:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765108/real 1775765108] req@ffff8f9970a76d80 x1862022967089152/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765108.422379:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967089152/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765108.422387:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765108.422391:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765108.422412:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967089152:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765109.317843:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765110.341998:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765111.365890:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765112.390250:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765113.413912:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765113.541876:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765113.541893:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765113.541898:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765113.541907:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1735 00000100:00080000:3.0:1775765113.541918:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765113.541926:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765113.541977:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967089280/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765113.542000:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765113.542003:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765113.542005:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765113.542009:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1735 00000100:00080000:3.0:1775765113.542013:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765113.542017:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765113.542026:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765113.542040:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967089408/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765113.542041:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967089280:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775765113.542058:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765113.542062:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765113.542064:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765113.542069:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1735 00000100:00080000:3.0:1775765113.542074:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765113.542079:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765113.542086:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765113.542102:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967089536/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765113.542113:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765113.542117:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765113.542120:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765113.542125:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1735 00000100:00080000:3.0:1775765113.542131:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765113.542136:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765113.542143:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765113.542151:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967089408:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775765113.542159:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967089664/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765113.542181:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967089536:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765113.542200:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967089664:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775765114.437957:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765115.461909:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765116.485976:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775765116.613990:0:2368:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:1.0:1775765116.614026:0:2368:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775765116.614040:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775765116.614082:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775765116.614091:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775765116.614099:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:2.0:1775765116.614133:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765113/real 1775765116] req@ffff8f9970ede300 x1862022967089664/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765116.614163:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967089664/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765116.614176:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765116.614181:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765116.614186:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967089664:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765116.614195:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765113/real 1775765116] req@ffff8f9970edf800 x1862022967089408/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765116.614211:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967089408/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765116.614267:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765116.614272:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765116.614275:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967089408:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765116.614284:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765113/real 1775765116] req@ffff8f9970ede680 x1862022967089536/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765116.614310:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967089536/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765116.614318:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765116.614322:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765116.614326:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967089536:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765116.614335:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765113/real 1775765116] req@ffff8f9970edc700 x1862022967089280/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765116.614349:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967089280/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765116.614357:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765116.614371:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765116.614373:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967089280:192.168.202.146@tcp:250:kworker.0 00010000:00010000:2.0:1775765117.509870:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765118.533887:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765118.661813:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765118.661824:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765118.661828:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765118.661838:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1740 00000100:00080000:3.0:1775765118.661848:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765118.661853:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765118.661885:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967089792/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765118.661913:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765118.661918:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765118.661920:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765118.661924:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1740 00000100:00080000:3.0:1775765118.661929:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765118.661933:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765118.661942:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765118.661950:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967089792:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765118.661955:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967089920/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765118.662009:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775765118.662022:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765118.662025:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765118.662027:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765118.662035:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1740 00000100:00080000:3.0:1775765118.662041:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765118.662046:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765118.662058:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765118.662072:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967090048/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765118.662081:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775765118.662082:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765118/real 1775765118] req@ffff8f9970eded80 x1862022967089792/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765118.662086:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765118.662089:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765118.662097:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1740 00000100:00080000:3.0:1775765118.662103:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765118.662107:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765118.662111:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765118.662118:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967089792/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765118.662120:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967090176/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765118.662155:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765118.662162:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765118.662169:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967089792:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765118.662195:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967089920:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765118.662216:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765118.662226:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765118/real 1775765118] req@ffff8f9970edc000 x1862022967089920/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765118.662240:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967089920/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765118.662248:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765118.662253:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765118.662256:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967089920:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765118.662262:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967090048:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765118.662283:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765118.662293:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765118/real 1775765118] req@ffff8f9970edce00 x1862022967090048/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765118.662306:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967090048/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765118.662339:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765118.662343:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765118.662345:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967090048:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765118.662351:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967090176:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765118.662373:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765118.662384:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765118/real 1775765118] req@ffff8f9970edd500 x1862022967090176/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765118.662395:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967090176/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765118.662419:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765118.662422:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765118.662425:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967090176:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775765119.557904:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765120.581901:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765121.605907:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765122.629917:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765123.653871:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765123.781821:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765123.781830:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765123.781834:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765123.781842:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1745 00000100:00080000:3.0:1775765123.781889:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765123.781895:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765123.781928:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddc00 x1862022967090304/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765123.781950:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765123.781953:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765123.781954:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765123.781957:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1745 00000100:00080000:3.0:1775765123.781960:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765123.781962:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765123.781969:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765123.781977:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf100 x1862022967090432/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765123.781984:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765123.781987:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765123.781988:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765123.781991:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1745 00000100:00080000:3.0:1775765123.781992:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765123.781995:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765123.781998:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765123.782002:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967090304:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765123.782008:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967090560/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765123.782016:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765123.782028:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765123.782030:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765123.782032:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1745 00000100:00080000:3.0:1775765123.782035:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765123.782037:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765123.782041:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765123.782048:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967090688/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765123.782076:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765123.782120:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765123/real 1775765123] req@ffff8f9970eddc00 x1862022967090304/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765123.782138:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddc00 x1862022967090304/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765123.782171:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765123.782176:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765123.782180:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967090304:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765123.782187:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967090432:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765123.782208:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765123.782219:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765123/real 1775765123] req@ffff8f9970edf100 x1862022967090432/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765123.782232:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf100 x1862022967090432/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765123.782239:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765123.782242:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765123.782245:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967090432:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765123.782268:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967090560:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765123.782285:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765123.782295:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765123/real 1775765123] req@ffff8f9970edd500 x1862022967090560/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765123.782307:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967090560/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765123.782315:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765123.782318:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765123.782321:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967090560:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765123.782343:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967090688:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765123.782388:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765123.782398:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765123/real 1775765123] req@ffff8f9970edce00 x1862022967090688/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765123.782409:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967090688/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765123.782416:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765123.782420:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765123.782423:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967090688:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765124.677952:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765125.702072:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765126.726088:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765127.749910:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765128.773924:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765128.901863:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765128.901874:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765128.901880:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765128.901891:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1750 00000100:00080000:3.0:1775765128.901904:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765128.901911:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765128.901958:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967090816/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765128.901989:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765128.901993:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765128.901994:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765128.901998:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1750 00000100:00080000:3.0:1775765128.902000:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765128.902004:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765128.902013:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765128.902022:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967090944/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765128.902101:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967090816:192.168.202.146@tcp:250:kworker.0 00000800:00000100:2.0:1775765128.902203:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775765128.902286:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765128.902292:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765128.902295:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765128.902321:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1750 00000100:00100000:2.0:1775765128.902344:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765128/real 1775765128] req@ffff8f9970edc000 x1862022967090816/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765128.902351:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765128.902362:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765128.902369:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967090816/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1775765128.902375:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1775765128.902380:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765128.902385:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765128.902390:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967090816:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765128.902412:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967091072/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765128.902416:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967090944:192.168.202.146@tcp:38:kworker.0 00000100:00080000:3.0:1775765128.902430:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765128.902433:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765128.902435:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765128.902440:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1750 00000100:00080000:3.0:1775765128.902444:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765128.902448:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765128.902453:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765128.902463:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967091200/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765128.902572:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765128.902597:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765128/real 1775765128] req@ffff8f9970eded80 x1862022967090944/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765128.902610:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967090944/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765128.902618:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765128.902621:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765128.902624:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967090944:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765128.902646:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967091072:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765128.902659:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765128.902665:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765128/real 1775765128] req@ffff8f9970edc700 x1862022967091072/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765128.902673:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967091072/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765128.902678:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765128.902680:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765128.902682:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967091072:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765128.902686:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967091200:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765128.902696:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765128.902701:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765128/real 1775765128] req@ffff8f9970ede680 x1862022967091200/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765128.902731:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967091200/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765128.902765:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765128.902768:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765128.902770:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967091200:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765129.797934:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765130.821887:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765131.845957:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765132.869971:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765133.893963:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765134.021799:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765134.021808:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765134.021812:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765134.021822:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1755 00000100:00080000:3.0:1775765134.021870:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765134.021878:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765134.021916:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967091328/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765134.021943:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765134.021947:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765134.021948:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765134.021952:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1755 00000100:00080000:3.0:1775765134.021955:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765134.021958:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765134.021967:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765134.021976:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967091456/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765134.021983:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765134.021985:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765134.021987:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765134.021989:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1755 00000100:00080000:3.0:1775765134.021991:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765134.021993:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765134.021997:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765134.022003:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967091584/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765134.022011:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775765134.022017:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967091328:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775765134.022027:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765134.022028:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765134.022032:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1755 00000100:00080000:3.0:1775765134.022035:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765134.022037:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765134.022041:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765134.022050:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967091712/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765134.022085:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765134.022126:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765134/real 1775765134] req@ffff8f9970edf800 x1862022967091328/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765134.022141:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967091328/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765134.022149:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765134.022152:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765134.022155:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967091328:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765134.022160:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967091456:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765134.022177:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765134.022185:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765134/real 1775765134] req@ffff8f9970ede300 x1862022967091456/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765134.022217:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967091456/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765134.022223:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765134.022225:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765134.022227:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967091456:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765134.022246:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967091584:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765134.022301:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765134.022320:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765134/real 1775765134] req@ffff8f9970edd880 x1862022967091584/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765134.022333:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967091584/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765134.022342:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765134.022344:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765134.022346:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967091584:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765134.022351:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967091712:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765134.022365:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765134.022372:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765134/real 1775765134] req@ffff8f9970edfb80 x1862022967091712/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765134.022379:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967091712/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765134.022402:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765134.022404:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765134.022407:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967091712:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765134.917856:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765135.941887:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765136.965991:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765137.989979:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765139.013953:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765139.142098:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765139.142117:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765139.142124:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765139.142142:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1760 00000100:00080000:1.0:1775765139.142163:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765139.142174:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765139.142252:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967091840/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765139.142605:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765139.142612:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765139.142617:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765139.142628:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1760 00000100:00080000:1.0:1775765139.142642:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765139.142651:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765139.142668:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765139.142690:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967091840:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775765139.142708:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967091968/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765139.142722:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765139.142726:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765139.142729:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765139.142734:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1760 00000100:00080000:1.0:1775765139.142799:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765139.142806:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765139.142814:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765139.142839:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967092096/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765139.142839:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775765139.142856:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765139.142861:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765139.142864:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765139.142869:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1760 00000100:00080000:1.0:1775765139.142902:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775765139.142902:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765139/real 1775765139] req@ffff8f994f2e4700 x1862022967091840/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765139.142908:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765139.142915:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765139.142926:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967091840/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765139.142929:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967092224/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765139.142960:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765139.142967:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765139.142974:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967091840:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765139.143011:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967091968:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765139.143060:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765139.143078:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765139/real 1775765139] req@ffff8f994f2e6680 x1862022967091968/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765139.143093:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967091968/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765139.143170:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765139.143176:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765139.143181:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967091968:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765139.143203:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967092096:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765139.143258:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765139.143272:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765139/real 1775765139] req@ffff8f994f2e5f80 x1862022967092096/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765139.143302:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967092096/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765139.143311:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765139.143314:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765139.143318:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967092096:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765139.143324:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967092224:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765139.143344:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765139.143353:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765139/real 1775765139] req@ffff8f994f2e6a00 x1862022967092224/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765139.143364:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967092224/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765139.143372:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765139.143375:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765139.143399:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967092224:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765140.037982:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765141.061880:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765142.085867:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765143.109865:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765144.133956:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765144.261841:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765144.261850:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765144.261855:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765144.261865:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1765 00000100:00080000:3.0:1775765144.261878:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765144.261884:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765144.261922:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967092352/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765144.261950:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765144.261955:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765144.261958:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765144.261963:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1765 00000100:00080000:3.0:1775765144.261969:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765144.261974:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765144.261979:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967092352:192.168.202.146@tcp:250:kworker.0 00000008:00100000:3.0:1775765144.261986:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765144.262004:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967092480/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765144.262035:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775765144.262057:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765144.262061:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765144.262063:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765144.262069:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1765 00000100:00080000:3.0:1775765144.262074:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775765144.262075:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765144/real 1775765144] req@ffff8f9970edea00 x1862022967092352/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765144.262084:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765144.262094:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765144.262099:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967092352/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765144.262108:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765144.262113:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765144.262113:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967092608/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765144.262117:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967092352:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775765144.262133:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765144.262137:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765144.262140:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765144.262143:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967092480:192.168.202.146@tcp:38:kworker.0 00000100:00080000:3.0:1775765144.262148:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1765 00000100:00080000:3.0:1775765144.262154:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765144.262174:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765144.262181:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765144.262186:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765144.262197:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967092736/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765144.262200:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765144/real 1775765144] req@ffff8f9970edf480 x1862022967092480/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765144.262227:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967092480/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765144.262234:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765144.262237:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765144.262239:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967092480:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765144.262268:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967092608:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765144.262291:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765144.262303:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765144/real 1775765144] req@ffff8f9970edca80 x1862022967092608/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765144.262315:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967092608/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765144.262324:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765144.262328:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765144.262331:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967092608:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765144.262346:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967092736:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765144.262362:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765144.262370:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765144/real 1775765144] req@ffff8f9970edc380 x1862022967092736/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765144.262377:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967092736/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765144.262382:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765144.262385:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765144.262387:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967092736:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775765145.157914:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765146.181911:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765147.205971:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765148.229888:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765149.253935:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765149.381837:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765149.381845:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765149.381849:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765149.381858:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1770 00000100:00080000:1.0:1775765149.381868:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765149.381874:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765149.381904:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967092864/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765149.381930:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765149.381934:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765149.381935:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765149.381938:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1770 00000100:00080000:1.0:1775765149.381941:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765149.381943:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765149.381951:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765149.381961:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967092992/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765149.381972:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967092864:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775765149.382012:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765149.382015:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765149.382016:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765149.382020:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1770 00000100:00080000:1.0:1775765149.382039:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765149.382042:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765149.382046:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775765149.382048:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765149.382058:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967093120/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765149.382065:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765149.382068:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765149.382069:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765149.382072:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1770 00000100:00080000:1.0:1775765149.382074:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765149.382076:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765149.382080:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765149.382085:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765149/real 1775765149] req@ffff8f994f2e4a80 x1862022967092864/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765149.382089:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967093248/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765149.382203:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967092864/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765149.382211:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765149.382214:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765149.382217:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967092864:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765149.382234:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967092992:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775765149.382252:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765149.382260:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765149/real 1775765149] req@ffff8f994f2e5180 x1862022967092992/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765149.382276:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967092992/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765149.382282:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765149.382284:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765149.382286:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967092992:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775765149.382290:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967093120:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775765149.382402:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765149.382412:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765149/real 1775765149] req@ffff8f994f2e7800 x1862022967093120/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765149.382421:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967093120/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765149.382510:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765149.382513:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765149.382515:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967093120:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775765149.382520:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967093248:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775765149.382538:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765149.382546:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765149/real 1775765149] req@ffff8f994f2e4000 x1862022967093248/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765149.382561:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967093248/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765149.382567:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765149.382569:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765149.382571:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967093248:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775765150.277911:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765151.302014:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765152.325890:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765153.349871:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765154.374046:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765154.501956:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765154.501972:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765154.502161:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765154.502174:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1776 00000100:00080000:1.0:1775765154.502188:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765154.502195:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765154.502238:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967093376/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765154.502268:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765154.502272:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765154.502273:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765154.502277:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1776 00000100:00080000:1.0:1775765154.502280:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765154.502283:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765154.502292:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765154.502305:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967093504/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765154.502313:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765154.502316:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765154.502317:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765154.502320:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1776 00000100:00080000:1.0:1775765154.502323:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765154.502325:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765154.502329:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765154.502334:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967093376:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775765154.502340:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967093632/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765154.502379:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765154.502382:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765154.502383:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765154.502387:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1776 00000100:00080000:1.0:1775765154.502392:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765154.502395:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765154.502399:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775765154.502404:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765154.502412:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967093760/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765154.502468:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765154/real 1775765154] req@ffff8f994f2e6d80 x1862022967093376/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765154.502494:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967093376/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765154.502509:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765154.502516:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765154.502522:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967093376:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765154.502533:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967093504:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775765154.502558:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765154.502572:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765154/real 1775765154] req@ffff8f994f2e4000 x1862022967093504/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765154.502584:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967093504/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765154.502603:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765154.502607:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765154.502611:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967093504:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775765154.502645:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967093632:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775765154.502671:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765154.502683:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765154/real 1775765154] req@ffff8f994f2e7800 x1862022967093632/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765154.502697:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967093632/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765154.502707:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765154.502711:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765154.502715:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967093632:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775765154.502722:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967093760:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775765154.502783:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765154.502797:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765154/real 1775765154] req@ffff8f994f2e5180 x1862022967093760/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765154.502813:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967093760/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765154.502822:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765154.502826:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765154.502830:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967093760:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765155.398026:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765156.421926:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765157.445919:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765158.469881:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765159.493972:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765159.621895:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765159.621908:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765159.621916:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765159.621929:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1781 00000100:00080000:3.0:1775765159.621943:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765159.621951:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765159.621998:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddf80 x1862022967093888/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765159.622065:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765159.622070:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765159.622073:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765159.622078:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1781 00000100:00080000:3.0:1775765159.622083:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765159.622088:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765159.622100:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765159.622125:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd180 x1862022967094016/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765159.622136:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967093888:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775765159.622192:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765159.622196:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765159.622198:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765159.622205:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1781 00000100:00080000:3.0:1775765159.622211:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765159.622215:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765159.622222:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765159.622223:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765159.622249:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967094144/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765159.622262:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765159.622268:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765159.622270:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765159.622275:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1781 00000100:00080000:3.0:1775765159.622279:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765159.622283:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765159.622288:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765159.622302:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765159/real 1775765159] req@ffff8f9970eddf80 x1862022967093888/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765159.622307:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967094272/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765159.622375:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddf80 x1862022967093888/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765159.622399:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765159.622409:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765159.622418:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967093888:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765159.622457:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967094016:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765159.622508:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765159.622534:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765159/real 1775765159] req@ffff8f9970edd180 x1862022967094016/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765159.622559:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd180 x1862022967094016/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765159.622576:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765159.622582:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765159.622588:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967094016:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765159.622598:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967094144:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765159.622629:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765159.622643:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765159/real 1775765159] req@ffff8f9970edc380 x1862022967094144/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765159.622657:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967094144/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765159.622668:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765159.622672:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765159.622676:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967094144:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765159.622682:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967094272:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765159.622700:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765159.622710:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765159/real 1775765159] req@ffff8f9970edca80 x1862022967094272/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765159.622723:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967094272/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765159.622732:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765159.622736:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765159.622773:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967094272:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765160.517918:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765161.541993:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765162.565903:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765163.589923:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765164.614003:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765164.742016:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765164.742025:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765164.742029:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765164.742038:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1786 00000100:00080000:0.0:1775765164.742048:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765164.742053:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765164.742087:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967094400/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765164.742126:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765164.742129:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765164.742131:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765164.742135:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1786 00000100:00080000:0.0:1775765164.742139:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765164.742142:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765164.742150:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765164.742162:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967094528/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765164.742173:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775765164.742174:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967094400:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775765164.742177:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765164.742184:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765164.742194:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1786 00000100:00080000:0.0:1775765164.742235:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765164.742240:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765164.742244:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765164.742254:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967094656/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765164.742262:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765164.742265:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765164.742266:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765164.742272:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1786 00000100:00080000:0.0:1775765164.742275:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765164.742277:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765164.742282:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765164.742282:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765164.742298:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967094784/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765164.742344:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765164/real 1775765164] req@ffff8f9970a74700 x1862022967094400/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765164.742369:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967094400/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765164.742380:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765164.742384:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765164.742388:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967094400:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765164.742419:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967094528:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765164.742460:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765164.742475:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765164/real 1775765164] req@ffff8f9970a75c00 x1862022967094528/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765164.742493:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967094528/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765164.742499:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765164.742501:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765164.742504:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967094528:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765164.742509:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967094656:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765164.742525:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765164.742532:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765164/real 1775765164] req@ffff8f9970a74380 x1862022967094656/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765164.742545:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967094656/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765164.742551:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765164.742553:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765164.742555:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967094656:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765164.742577:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967094784:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765164.742594:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765164.742602:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765164/real 1775765164] req@ffff8f9970a76d80 x1862022967094784/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765164.742612:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967094784/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765164.742618:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765164.742621:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765164.742623:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967094784:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765165.637892:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765166.661907:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765167.685870:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765168.709966:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765169.734091:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765169.861936:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765169.861955:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765169.861964:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765169.862002:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1791 00000100:00080000:3.0:1775765169.862065:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765169.862075:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765169.862134:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967094912/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765169.862166:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765169.862170:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765169.862172:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765169.862176:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1791 00000100:00080000:3.0:1775765169.862179:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765169.862183:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765169.862195:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765169.862207:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967095040/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765169.862217:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765169.862221:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765169.862222:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765169.862229:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1791 00000100:00100000:2.0:1775765169.862231:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967094912:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775765169.862232:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765169.862242:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765169.862248:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765169.862266:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967095168/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765169.862280:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765169.862286:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765169.862289:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765169.862294:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1791 00000100:00080000:3.0:1775765169.862311:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775765169.862326:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775765169.862330:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765169.862349:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765169.862364:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967095296/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765169.862403:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765169/real 1775765169] req@ffff8f9970edf480 x1862022967094912/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765169.862496:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967094912/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765169.862512:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765169.862518:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765169.862523:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967094912:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765169.862560:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967095040:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765169.862590:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765169.862603:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765169/real 1775765169] req@ffff8f9970edea00 x1862022967095040/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765169.862614:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967095040/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765169.862621:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765169.862624:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765169.862627:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967095040:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765169.862632:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967095168:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765169.862648:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765169.862655:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765169/real 1775765169] req@ffff8f9970edfb80 x1862022967095168/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765169.862664:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967095168/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765169.862695:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765169.862699:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765169.862702:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967095168:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765169.862715:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967095296:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765169.862732:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765169.862764:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765169/real 1775765169] req@ffff8f9970edd880 x1862022967095296/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765169.862774:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967095296/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765169.862781:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765169.862784:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765169.862786:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967095296:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765170.758129:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765171.781999:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765172.805949:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765173.830035:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765174.853942:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765174.981909:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765174.981935:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765174.981941:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765174.981954:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1796 00000100:00080000:1.0:1775765174.981967:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765174.981974:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765174.982019:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967095424/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765174.982046:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775765174.982084:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967095424:192.168.202.146@tcp:250:kworker.0 00000800:00000100:2.0:1775765174.982161:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775765174.982179:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765174.982184:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765174.982195:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1796 00000100:00080000:1.0:1775765174.982208:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765174.982214:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765174.982220:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765174/real 1775765174] req@ffff8f994f2e4a80 x1862022967095424/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1775765174.982231:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765174.982241:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967095424/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765174.982272:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967095552/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765174.982298:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765174.982299:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765174.982313:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765174.982313:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00080000:1.0:1775765174.982320:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765174.982326:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967095424:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775765174.982329:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1796 00000100:00080000:1.0:1775765174.982340:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765174.982346:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765174.982353:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765174.982366:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967095552:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775765174.982368:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967095680/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765174.982398:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765174.982401:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765174.982402:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765174.982409:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1796 00000100:00080000:1.0:1775765174.982415:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765174.982418:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:2.0:1775765174.982429:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:1.0:1775765174.982453:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765174.982462:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765174/real 1775765174] req@ffff8f994f2e6a00 x1862022967095552/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765174.982468:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967095808/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765174.982507:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967095552/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765174.982516:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765174.982530:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765174.982534:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967095552:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765174.982555:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967095680:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765174.982599:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765174.982611:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765174/real 1775765174] req@ffff8f994f2e5f80 x1862022967095680/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765174.982624:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967095680/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765174.982631:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765174.982635:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765174.982638:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967095680:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765174.982703:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967095808:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765174.982725:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765174.982735:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765174/real 1775765174] req@ffff8f994f2e6680 x1862022967095808/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765174.982788:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967095808/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765174.982795:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765174.982798:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765174.982801:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967095808:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775765175.877913:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765176.901977:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765177.926078:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765178.950029:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765179.973918:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765180.101908:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765180.101920:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765180.101926:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765180.101938:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1801 00000100:00080000:1.0:1775765180.101952:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765180.101960:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765180.102008:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967095936/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765180.102038:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765180.102041:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765180.102043:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765180.102047:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1801 00000100:00080000:1.0:1775765180.102050:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765180.102053:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765180.102064:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765180.102076:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967096064/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765180.102081:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967095936:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775765180.102091:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765180.102095:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765180.102096:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765180.102144:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1801 00000100:00080000:1.0:1775765180.102149:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765180.102153:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765180.102160:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765180.102174:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967096192/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765180.102185:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765180.102189:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765180.102190:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765180.102195:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1801 00000100:00080000:1.0:1775765180.102199:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765180.102203:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765180.102211:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765180.102228:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967096320/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765180.102259:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967096064:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765180.102310:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967096192:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765180.102331:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967096320:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775765180.997990:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765182.021938:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765183.045882:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775765183.174026:0:2367:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:2.0:1775765183.174072:0:2367:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775765183.174091:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:2.0:1775765183.174137:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:3.0:1775765183.174305:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765180/real 1775765183] req@ffff8f994f2e7b80 x1862022967096064/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1775765183.174306:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:2.0:1775765183.174332:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:3.0:1775765183.174353:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967096064/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765183.174370:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765183.174376:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765183.174383:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967096064:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775765183.174395:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765180/real 1775765183] req@ffff8f994f2e6300 x1862022967096192/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765183.174410:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967096192/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765183.174418:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765183.174421:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765183.174425:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967096192:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775765183.174435:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765180/real 1775765183] req@ffff8f994f2e4e00 x1862022967096320/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765183.174449:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967096320/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765183.174456:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765183.174460:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765183.174464:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967096320:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775765183.174470:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765180/real 1775765183] req@ffff8f994f2e4700 x1862022967095936/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765183.174480:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967095936/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765183.174487:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765183.174491:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765183.174495:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967095936:192.168.202.146@tcp:250:kworker.0 00010000:00010000:2.0:1775765184.069875:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765185.093894:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765185.221862:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765185.221870:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765185.221874:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765185.221883:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1806 00000100:00080000:1.0:1775765185.221893:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765185.221900:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765185.221940:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7100 x1862022967096448/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765185.221982:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765185.221985:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765185.221986:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765185.221991:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1806 00000100:00080000:1.0:1775765185.221994:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765185.221999:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765185.222007:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765185.222021:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967096576/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765185.222022:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967096448:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775765185.222036:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765185.222039:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765185.222041:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765185.222043:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1806 00000100:00080000:1.0:1775765185.222048:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765185.222050:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765185.222054:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765185.222064:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967096704/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765185.222074:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765185.222078:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765185.222080:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765185.222083:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1806 00000800:00000100:3.0:1775765185.222089:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775765185.222104:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765185.222110:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765185.222116:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765185.222130:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967096832/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765185.222136:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765185/real 1775765185] req@ffff8f994f2e7100 x1862022967096448/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765185.222160:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7100 x1862022967096448/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765185.222174:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765185.222179:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765185.222184:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967096448:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765185.222210:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967096576:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775765185.222237:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765185.222251:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765185/real 1775765185] req@ffff8f994f2e4700 x1862022967096576/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765185.222262:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967096576/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765185.222270:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765185.222273:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765185.222276:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967096576:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775765185.222282:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967096704:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775765185.222302:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765185.222312:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765185/real 1775765185] req@ffff8f994f2e4e00 x1862022967096704/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765185.222325:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967096704/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765185.222334:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765185.222338:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765185.222341:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967096704:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775765185.222347:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967096832:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775765185.222367:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765185.222377:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765185/real 1775765185] req@ffff8f994f2e6300 x1862022967096832/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765185.222389:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967096832/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765185.222397:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765185.222401:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765185.222404:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967096832:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765186.118010:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765187.141920:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765188.165940:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765189.189872:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765190.213975:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765190.341923:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765190.341938:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765190.341944:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765190.341959:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1811 00000100:00080000:3.0:1775765190.341973:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765190.341980:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765190.342032:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967096960/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765190.342069:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765190.342074:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765190.342077:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765190.342082:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1811 00000100:00080000:3.0:1775765190.342086:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765190.342093:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765190.342107:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765190.342125:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967097088/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765190.342137:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967096960:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775765190.342143:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765190.342160:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765190.342162:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765190.342168:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1811 00000100:00080000:3.0:1775765190.342171:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765190.342176:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765190.342182:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765190.342196:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967097216/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765190.342202:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765190.342206:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765190.342207:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765190.342210:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1811 00000100:00080000:3.0:1775765190.342212:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765190.342216:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765190.342219:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765190.342227:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967097344/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765190.342263:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765190.342331:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765190/real 1775765190] req@ffff8f9970ede300 x1862022967096960/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765190.342359:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967096960/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765190.342374:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765190.342381:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765190.342387:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967096960:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765190.342420:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967097088:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765190.342452:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765190.342467:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765190/real 1775765190] req@ffff8f9970edf800 x1862022967097088/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765190.342481:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967097088/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765190.342491:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765190.342495:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765190.342587:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967097088:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765190.342596:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967097216:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765190.342621:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765190.342635:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765190/real 1775765190] req@ffff8f9970ede680 x1862022967097216/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765190.342650:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967097216/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765190.342660:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765190.342663:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765190.342667:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967097216:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765190.342674:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967097344:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765190.342693:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765190.342703:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765190/real 1775765190] req@ffff8f9970edc700 x1862022967097344/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765190.342717:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967097344/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765190.342726:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765190.342730:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765190.342785:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967097344:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765191.237903:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765192.262001:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765193.285891:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765194.309895:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765195.333920:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765195.461832:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765195.461841:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765195.461845:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765195.461852:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1817 00000100:00080000:0.0:1775765195.461862:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765195.461867:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765195.461898:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967097472/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765195.461923:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765195.461965:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765195.461966:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765195.461970:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1817 00000100:00080000:0.0:1775765195.461974:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765195.461980:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765195.461982:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967097472:192.168.202.146@tcp:250:kworker.0 00000008:00100000:0.0:1775765195.461988:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765195.462000:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967097600/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765195.462008:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765195.462011:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765195.462012:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765195.462015:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1817 00000100:00080000:0.0:1775765195.462018:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765195.462021:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765195.462023:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765195.462030:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967097728/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765195.462036:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765195.462039:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765195.462040:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1775765195.462043:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775765195.462047:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1817 00000100:00080000:0.0:1775765195.462050:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765195.462052:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765195.462066:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765195.462074:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967097856/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765195.462089:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765195/real 1775765195] req@ffff8f9970a76a00 x1862022967097472/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765195.462104:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967097472/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765195.462111:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765195.462114:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765195.462118:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967097472:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765195.462138:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967097600:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765195.462152:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765195.462160:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765195/real 1775765195] req@ffff8f9970a77800 x1862022967097600/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765195.462168:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967097600/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765195.462172:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765195.462174:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765195.462176:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967097600:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765195.462180:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967097728:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765195.462190:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765195.462195:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765195/real 1775765195] req@ffff8f9970a77480 x1862022967097728/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765195.462221:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967097728/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765195.462227:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765195.462229:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765195.462231:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967097728:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765195.462235:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967097856:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765195.462246:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765195.462252:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765195/real 1775765195] req@ffff8f9970a75180 x1862022967097856/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765195.462260:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967097856/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765195.462265:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765195.462266:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765195.462268:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967097856:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765196.357901:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765197.381845:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765198.405998:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765199.429920:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765200.453911:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765200.581831:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765200.581842:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765200.581846:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765200.581857:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1822 00000100:00080000:3.0:1775765200.581884:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765200.581890:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765200.581924:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967097984/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765200.581948:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765200.581951:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765200.581952:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765200.581957:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1822 00000100:00080000:3.0:1775765200.581960:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765200.581964:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765200.581972:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765200.581981:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967097984:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765200.581984:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967098112/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765200.581998:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765200.582000:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765200.582001:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765200.582004:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1822 00000100:00080000:3.0:1775765200.582006:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765200.582008:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765200.582012:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765200.582020:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967098240/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765200.582033:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765200.582036:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765200.582038:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765200.582040:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1822 00000100:00080000:3.0:1775765200.582062:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765200.582065:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765200.582069:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765200.582080:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967098368/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765200.582082:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765200.582170:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765200/real 1775765200] req@ffff8f9970eded80 x1862022967097984/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765200.582280:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967097984/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765200.582293:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765200.582298:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765200.582304:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967097984:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765200.582344:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967098112:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765200.582371:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765200.582404:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765200/real 1775765200] req@ffff8f9970edc000 x1862022967098112/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765200.582420:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967098112/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765200.582430:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765200.582434:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765200.582438:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967098112:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765200.582445:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967098240:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765200.582471:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765200.582484:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765200/real 1775765200] req@ffff8f9970edce00 x1862022967098240/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765200.582499:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967098240/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765200.582537:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765200.582542:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765200.582546:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967098240:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765200.582553:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967098368:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765200.582594:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765200.582606:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765200/real 1775765200] req@ffff8f9970edd500 x1862022967098368/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765200.582619:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967098368/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765200.582628:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765200.582631:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765200.582635:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967098368:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765201.477927:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765202.501886:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765203.525874:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765204.549889:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765205.573980:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765205.702057:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765205.702072:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765205.702078:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765205.702095:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1827 00000100:00080000:3.0:1775765205.702111:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765205.702121:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765205.702178:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf100 x1862022967098496/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765205.702220:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765205.702225:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765205.702228:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765205.702235:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1827 00000100:00080000:3.0:1775765205.702241:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765205.702247:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765205.702263:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765205.702274:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967098496:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765205.702283:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddc00 x1862022967098624/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765205.702296:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765205.702300:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765205.702302:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765205.702306:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1827 00000100:00080000:3.0:1775765205.702311:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765205.702315:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765205.702322:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765205.702334:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967098752/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765205.702345:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765205.702349:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765205.702350:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765205.702354:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1827 00000100:00080000:3.0:1775765205.702358:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765205.702362:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765205.702366:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765205.702374:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967098880/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765205.702467:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765205.702502:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765205/real 1775765205] req@ffff8f9970edf100 x1862022967098496/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765205.702517:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf100 x1862022967098496/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765205.702526:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765205.702537:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765205.702541:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967098496:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765205.702565:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967098624:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775765205.702582:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765205.702625:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765205/real 1775765205] req@ffff8f9970eddc00 x1862022967098624/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765205.702639:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddc00 x1862022967098624/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765205.702648:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765205.702652:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765205.702656:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967098624:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775765205.702662:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967098752:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765205.702704:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765205.702713:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765205/real 1775765205] req@ffff8f9970edd500 x1862022967098752/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765205.702721:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967098752/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765205.702727:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765205.702729:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765205.702731:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967098752:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765205.702736:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967098880:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765205.702785:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765205.702792:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765205/real 1775765205] req@ffff8f9970edce00 x1862022967098880/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765205.702800:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967098880/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765205.702805:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765205.702808:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765205.702810:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967098880:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775765206.597908:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765207.621943:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765208.645908:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765209.669978:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765210.693921:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765210.821887:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765210.821895:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765210.821899:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765210.821907:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1832 00000100:00080000:1.0:1775765210.821916:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765210.821922:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765210.821954:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967099008/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765210.821975:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765210.821978:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765210.821979:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765210.821982:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1832 00000100:00080000:1.0:1775765210.821985:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775765210.822014:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967099008:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775765210.822032:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765210.822042:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765210.822057:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967099136/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765210.822066:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765210.822069:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765210.822070:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765210.822074:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1832 00000800:00000100:0.0:1775765210.822079:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775765210.822081:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765210.822086:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765210.822090:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765210.822099:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967099264/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765210.822109:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765210.822113:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1775765210.822116:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765210/real 1775765210] req@ffff8f994f2e7b80 x1862022967099008/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765210.822116:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765210.822126:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1832 00000100:00080000:1.0:1775765210.822132:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765210.822136:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765210.822142:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765210.822155:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967099392/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765210.822168:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967099008/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765210.822180:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765210.822185:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765210.822189:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967099008:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765210.822222:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967099136:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775765210.822243:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765210.822253:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765210/real 1775765210] req@ffff8f994f2e6680 x1862022967099136/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765210.822262:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967099136/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765210.822267:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765210.822269:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765210.822272:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967099136:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775765210.822276:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967099264:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765210.822287:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765210.822292:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765210/real 1775765210] req@ffff8f994f2e5f80 x1862022967099264/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765210.822299:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967099264/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765210.822322:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765210.822324:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765210.822326:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967099264:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765210.822330:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967099392:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765210.822342:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765210.822349:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765210/real 1775765210] req@ffff8f994f2e6a00 x1862022967099392/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765210.822356:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967099392/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765210.822361:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765210.822364:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765210.822366:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967099392:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775765211.717916:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765212.741885:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765213.766010:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765214.790001:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765215.813951:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765215.941870:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765215.941880:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765215.941885:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765215.941908:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1837 00000100:00080000:0.0:1775765215.941919:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765215.941927:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765215.941961:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967099520/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765215.941984:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765215.942024:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765215.942028:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765215.942036:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1837 00000100:00100000:2.0:1775765215.942036:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967099520:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775765215.942041:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765215.942044:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765215.942052:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765215.942060:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967099648/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765215.942069:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765215.942071:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765215.942072:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765215.942074:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1837 00000100:00080000:0.0:1775765215.942076:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765215.942078:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765215.942081:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765215.942086:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967099776/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765215.942091:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765215.942093:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765215.942094:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765215.942096:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1837 00000100:00080000:0.0:1775765215.942098:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765215.942100:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765215.942117:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765215.942126:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967099904/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765215.942199:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765215.942240:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765215/real 1775765215] req@ffff8f9970a77100 x1862022967099520/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765215.942256:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967099520/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765215.942263:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765215.942266:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765215.942269:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967099520:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765215.942291:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967099648:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765215.942305:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765215.942313:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765215/real 1775765215] req@ffff8f9970a76300 x1862022967099648/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765215.942321:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967099648/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765215.942326:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765215.942328:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765215.942330:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967099648:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765215.942340:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967099776:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765215.942351:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765215.942356:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765215/real 1775765215] req@ffff8f9970a75500 x1862022967099776/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765215.942384:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967099776/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765215.942390:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765215.942392:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765215.942394:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967099776:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765215.942397:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967099904:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765215.942409:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765215.942416:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765215/real 1775765215] req@ffff8f9970a75180 x1862022967099904/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765215.942423:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967099904/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765215.942428:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765215.942429:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765215.942431:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967099904:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765216.837911:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765217.861865:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765218.885917:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765219.909907:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765220.933914:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765221.061862:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765221.061870:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765221.061911:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765221.061919:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1842 00000100:00080000:3.0:1775765221.061932:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765221.061938:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765221.061971:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967100032/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765221.061996:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765221.061999:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765221.062001:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765221.062004:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1842 00000100:00080000:3.0:1775765221.062006:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765221.062009:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765221.062018:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765221.062046:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967100160/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765221.062061:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775765221.062071:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967100032:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775765221.062071:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765221.062075:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765221.062091:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1842 00000100:00080000:3.0:1775765221.062097:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765221.062105:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765221.062112:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775765221.062168:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765221.062218:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967100288/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765221.062231:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765221/real 1775765221] req@ffff8f9970edc000 x1862022967100032/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765221.062257:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967100032/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765221.062270:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765221.062276:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765221.062280:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967100032:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765221.062327:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967100160:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775765221.062352:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765221.062369:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765221/real 1775765221] req@ffff8f9970eded80 x1862022967100160/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765221.062378:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765221.062387:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765221.062390:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765221.062397:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1842 00000100:00080000:3.0:1775765221.062407:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765221.062411:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765221.062413:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967100160/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765221.062422:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765221.062426:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765221.062430:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967100160:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775765221.062466:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967100288:192.168.202.146@tcp:8:kworker.0 00000008:00100000:3.0:1775765221.062474:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765221.062489:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967100416/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765221.062613:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765221.062634:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765221/real 1775765221] req@ffff8f9970edc700 x1862022967100288/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765221.062648:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967100288/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765221.062655:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765221.062658:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765221.062661:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967100288:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765221.062682:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967100416:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765221.062694:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765221.062700:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765221/real 1775765221] req@ffff8f9970ede680 x1862022967100416/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765221.062707:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967100416/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765221.062712:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765221.062714:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765221.062716:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967100416:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765221.958000:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765222.981983:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765224.005968:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765225.030031:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765226.053929:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765226.181920:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765226.181929:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765226.181934:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765226.181943:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1847 00000100:00080000:3.0:1775765226.181952:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765226.181958:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765226.182007:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967100544/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765226.182033:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765226.182036:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765226.182037:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765226.182040:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1847 00000100:00080000:3.0:1775765226.182043:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765226.182045:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765226.182053:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765226.182061:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967100672/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765226.182069:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765226.182071:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765226.182073:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765226.182075:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1847 00000100:00080000:3.0:1775765226.182078:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775765226.182082:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967100544:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775765226.182083:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765226.182091:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765226.182097:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967100800/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765226.182114:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765226.182117:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765226.182118:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765226.182121:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1847 00000100:00080000:3.0:1775765226.182123:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765226.182125:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765226.182128:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765226.182136:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967100928/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765226.182155:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765226.182187:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765226/real 1775765226] req@ffff8f9970edf800 x1862022967100544/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765226.182254:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967100544/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765226.182262:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765226.182265:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765226.182268:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967100544:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765226.182274:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967100672:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775765226.182289:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765226.182298:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765226/real 1775765226] req@ffff8f9970ede300 x1862022967100672/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765226.182306:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967100672/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765226.182311:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765226.182313:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765226.182315:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967100672:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775765226.182346:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967100800:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765226.182358:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765226.182364:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765226/real 1775765226] req@ffff8f9970edd880 x1862022967100800/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765226.182371:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967100800/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765226.182392:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765226.182395:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765226.182397:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967100800:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765226.182414:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967100928:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765226.182429:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765226.182448:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765226/real 1775765226] req@ffff8f9970edfb80 x1862022967100928/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765226.182456:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967100928/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765226.182462:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765226.182464:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765226.182466:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967100928:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765227.077993:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765228.101874:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765229.126035:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765230.149940:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765231.173980:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765231.301861:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765231.301878:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765231.301887:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765231.301903:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1852 00000100:00080000:3.0:1775765231.301916:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765231.301925:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765231.301970:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967101056/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765231.302006:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765231.302011:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765231.302013:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765231.302018:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1852 00000100:00080000:3.0:1775765231.302022:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765231.302027:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765231.302049:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765231.302067:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967101184/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765231.302073:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967101056:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775765231.302088:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765231.302093:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765231.302095:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765231.302099:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1852 00000100:00080000:3.0:1775765231.302102:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765231.302105:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765231.302111:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765231.302124:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967101312/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765231.302143:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765231.302147:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765231.302149:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765231.302153:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1852 00000100:00080000:3.0:1775765231.302158:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765231.302162:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765231.302166:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775765231.302175:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765231.302182:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967101440/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765231.302244:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765231/real 1775765231] req@ffff8f9970edea00 x1862022967101056/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765231.302277:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967101056/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765231.302297:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765231.302305:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765231.302312:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967101056:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765231.302412:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967101184:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775765231.302444:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765231.302459:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765231/real 1775765231] req@ffff8f9970edf480 x1862022967101184/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765231.302474:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967101184/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765231.302483:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765231.302487:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765231.302491:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967101184:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775765231.302497:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967101312:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765231.302515:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765231.302525:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765231/real 1775765231] req@ffff8f9970edca80 x1862022967101312/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765231.302537:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967101312/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765231.302545:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765231.302548:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765231.302551:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967101312:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765231.302557:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967101440:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765231.302571:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765231.302580:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765231/real 1775765231] req@ffff8f9970edc380 x1862022967101440/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765231.302617:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967101440/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765231.302626:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765231.302629:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765231.302632:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967101440:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765232.197900:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765233.221883:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765234.245893:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765235.269918:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765236.293911:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765236.421912:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765236.421923:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765236.421928:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765236.421939:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1858 00000100:00080000:1.0:1775765236.421952:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765236.421959:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765236.422007:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967101568/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765236.422035:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765236.422038:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765236.422040:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765236.422043:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1858 00000100:00080000:1.0:1775765236.422045:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765236.422047:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765236.422057:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765236.422066:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967101696/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765236.422075:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775765236.422085:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967101568:192.168.202.146@tcp:250:kworker.0 00000800:00000100:0.0:1775765236.422160:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775765236.422199:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765236.422205:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775765236.422206:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765236/real 1775765236] req@ffff8f994f2e4a80 x1862022967101568/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765236.422225:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1858 00000100:00100000:0.0:1775765236.422234:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967101568/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765236.422239:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765236.422246:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765236.422252:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00080000:0.0:1775765236.422253:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765236.422261:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967101568:192.168.202.146@tcp:250:kworker.0 00000008:00100000:1.0:1775765236.422272:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765236.422288:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967101696:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775765236.422324:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967101824/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765236.422336:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775765236.422346:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765236.422349:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765236.422351:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765236.422356:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1858 00000100:00080000:1.0:1775765236.422360:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775765236.422366:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765236/real 1775765236] req@ffff8f994f2e5180 x1862022967101696/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765236.422370:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765236.422377:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765236.422400:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967101952/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765236.422401:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967101696/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765236.422556:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765236.422561:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765236.422565:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967101696:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775765236.422591:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967101824:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765236.422643:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765236.422665:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765236/real 1775765236] req@ffff8f994f2e7800 x1862022967101824/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765236.422683:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967101824/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765236.422692:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765236.422695:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765236.422699:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967101824:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765236.422706:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967101952:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765236.422728:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765236.422778:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765236/real 1775765236] req@ffff8f994f2e4000 x1862022967101952/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765236.422792:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967101952/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765236.422798:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765236.422800:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765236.422826:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967101952:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775765237.317898:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765238.341882:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765239.365916:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765240.389938:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765241.413878:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765241.541830:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765241.541838:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765241.541842:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765241.541850:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1863 00000100:00080000:3.0:1775765241.541859:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765241.541864:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765241.541895:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd180 x1862022967102080/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765241.541916:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765241.541919:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765241.541920:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765241.541924:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1863 00000100:00080000:3.0:1775765241.541926:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765241.541928:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765241.541960:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967102080:192.168.202.146@tcp:250:kworker.0 00000008:00100000:3.0:1775765241.541965:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765241.541978:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddf80 x1862022967102208/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765241.541990:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765241.541994:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765241.541996:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765241.542001:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1863 00000100:00080000:3.0:1775765241.542005:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765241.542009:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765241.542016:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775765241.542024:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765241.542031:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967102336/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765241.542042:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765241.542044:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765241.542045:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765241.542049:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1863 00000100:00080000:3.0:1775765241.542053:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765241.542055:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765241.542059:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765241.542067:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765241/real 1775765241] req@ffff8f9970edd180 x1862022967102080/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765241.542070:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967102464/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765241.542106:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd180 x1862022967102080/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765241.542115:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765241.542119:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765241.542122:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967102080:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765241.542141:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967102208:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775765241.542157:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765241.542165:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765241/real 1775765241] req@ffff8f9970eddf80 x1862022967102208/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765241.542173:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddf80 x1862022967102208/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765241.542178:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765241.542180:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765241.542181:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967102208:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775765241.542185:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967102336:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765241.542196:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765241.542222:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765241/real 1775765241] req@ffff8f9970edc380 x1862022967102336/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765241.542230:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967102336/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765241.542235:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765241.542237:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765241.542239:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967102336:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765241.542243:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967102464:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765241.542254:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765241.542285:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765241/real 1775765241] req@ffff8f9970edca80 x1862022967102464/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765241.542293:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967102464/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765241.542298:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765241.542300:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765241.542302:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967102464:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775765242.437883:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765243.461972:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765244.485951:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765245.509887:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765246.533878:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765246.661815:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765246.661825:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765246.661830:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765246.661840:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1868 00000100:00080000:3.0:1775765246.661850:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765246.661855:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765246.661892:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967102592/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765246.661920:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765246.661923:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765246.661925:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765246.661929:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1868 00000100:00080000:3.0:1775765246.661931:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765246.661934:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765246.661942:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765246.661952:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967102720/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765246.661961:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775765246.661966:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967102592:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775765246.661967:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765246.661971:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765246.661980:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1868 00000100:00080000:3.0:1775765246.661983:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765246.661985:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765246.661989:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765246.661998:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967102848/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765246.662011:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765246.662014:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765246.662016:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765246.662018:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1868 00000100:00080000:3.0:1775765246.662021:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765246.662024:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765246.662027:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765246.662035:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967102976/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765246.662087:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967102720:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775765246.662129:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967102848:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765246.662173:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967102976:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765247.558019:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765248.581991:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765249.605978:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1775765249.734421:0:2369:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:0.0:1775765249.734523:0:2369:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1775765249.734553:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775765249.734617:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775765249.734652:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775765249.734673:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:2.0:1775765249.734684:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765246/real 1775765249] req@ffff8f9970edea00 x1862022967102720/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765249.734728:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967102720/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765249.734790:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765249.734802:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765249.734812:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967102720:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765249.734824:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765246/real 1775765249] req@ffff8f9970edfb80 x1862022967102848/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765249.734842:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967102848/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765249.734850:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765249.734854:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765249.734858:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967102848:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765249.735458:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765246/real 1775765249] req@ffff8f9970edd880 x1862022967102976/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765249.735496:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967102976/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765249.735512:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765249.735518:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765249.735524:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967102976:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765249.735532:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765246/real 1775765249] req@ffff8f9970edf480 x1862022967102592/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765249.735547:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967102592/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765249.735556:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765249.735562:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765249.735568:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967102592:192.168.202.146@tcp:250:kworker.0 00010000:00010000:2.0:1775765250.629891:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765251.653887:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765251.781844:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765251.781854:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765251.781859:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765251.781870:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1873 00000100:00080000:2.0:1775765251.781881:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765251.781888:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765251.781922:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1f80 x1862022967103104/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765251.781980:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765251.781983:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765251.781984:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765251.781988:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1873 00000100:00080000:2.0:1775765251.781991:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765251.781997:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765251.782009:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765251.782011:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967103104:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765251.782026:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0a80 x1862022967103232/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765251.782043:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765251.782047:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765251.782050:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765251.782054:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1873 00000100:00080000:2.0:1775765251.782059:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765251.782064:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765251.782069:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775765251.782070:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765251.782085:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2d80 x1862022967103360/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765251.782096:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765251.782100:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765251.782101:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765251.782106:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1873 00000100:00100000:0.0:1775765251.782128:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765251/real 1775765251] req@ffff8f99703d1f80 x1862022967103104/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765251.782129:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765251.782137:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765251.782147:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765251.782179:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1f80 x1862022967103104/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765251.782192:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1775765251.782193:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0700 x1862022967103488/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765251.782201:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765251.782211:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967103104:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765251.782243:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967103232:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775765251.782268:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765251.782281:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765251/real 1775765251] req@ffff8f99703d0a80 x1862022967103232/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765251.782293:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0a80 x1862022967103232/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765251.782304:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765251.782332:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765251.782336:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967103232:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775765251.782342:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967103360:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765251.782364:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765251.782375:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765251/real 1775765251] req@ffff8f99703d2d80 x1862022967103360/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765251.782388:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2d80 x1862022967103360/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765251.782397:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765251.782401:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765251.782404:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967103360:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765251.782411:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967103488:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765251.782432:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765251.782443:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765251/real 1775765251] req@ffff8f99703d0700 x1862022967103488/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765251.782463:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0700 x1862022967103488/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765251.782471:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765251.782474:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765251.782477:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967103488:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765252.677893:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765253.701903:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765254.725973:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765255.749960:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765256.773909:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765256.901810:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765256.901820:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765256.901824:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765256.901834:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1878 00000100:00080000:2.0:1775765256.901844:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765256.901851:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765256.901895:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0e00 x1862022967103616/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765256.901922:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765256.901925:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765256.901927:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765256.901931:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1878 00000100:00080000:2.0:1775765256.901933:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765256.901936:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765256.901943:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765256.901963:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1500 x1862022967103744/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765256.901983:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967103616:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775765256.902026:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765256.902029:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765256.902030:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765256.902033:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1878 00000100:00080000:2.0:1775765256.902036:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765256.902038:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765256.902041:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765256.902052:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3800 x1862022967103872/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765256.902061:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765256.902065:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765256.902067:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765256.902069:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1878 00000100:00080000:2.0:1775765256.902072:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765256.902074:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765256.902078:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775765256.902099:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765256.902099:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2680 x1862022967104000/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765256.902156:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765256/real 1775765256] req@ffff8f99703d0e00 x1862022967103616/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765256.902174:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0e00 x1862022967103616/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765256.902182:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765256.902186:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765256.902189:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967103616:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765256.902216:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967103744:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775765256.902238:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765256.902249:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765256/real 1775765256] req@ffff8f99703d1500 x1862022967103744/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765256.902261:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1500 x1862022967103744/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765256.902269:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765256.902272:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765256.902279:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967103744:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775765256.902285:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967103872:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765256.902296:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765256.902302:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765256/real 1775765256] req@ffff8f99703d3800 x1862022967103872/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765256.902328:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3800 x1862022967103872/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765256.902334:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765256.902336:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765256.902339:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967103872:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765256.902343:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967104000:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765256.902358:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765256.902369:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765256/real 1775765256] req@ffff8f99703d2680 x1862022967104000/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765256.902378:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2680 x1862022967104000/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765256.902383:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765256.902384:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765256.902386:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967104000:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765257.797942:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765258.821945:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765259.846066:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765260.870008:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765261.893879:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765262.021936:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765262.021964:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765262.021970:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765262.021981:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1883 00000100:00080000:1.0:1775765262.021996:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765262.022019:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765262.022111:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967104128/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765262.022158:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765262.022162:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765262.022164:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765262.022168:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1883 00000100:00080000:1.0:1775765262.022172:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765262.022176:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765262.022185:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765262.022201:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967104256/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765262.022210:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765262.022214:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765262.022216:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765262.022219:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1883 00000100:00080000:1.0:1775765262.022222:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765262.022228:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765262.022230:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967104128:192.168.202.146@tcp:250:kworker.0 00000008:00100000:1.0:1775765262.022234:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765262.022247:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967104384/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765262.022257:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765262.022260:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765262.022262:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765262.022265:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1883 00000100:00080000:1.0:1775765262.022267:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765262.022270:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765262.022274:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765262.022283:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967104512/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765262.022333:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765262.022386:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765262/real 1775765262] req@ffff8f994f2e6d80 x1862022967104128/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765262.022427:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967104128/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765262.022494:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765262.022500:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765262.022506:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967104128:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765262.022516:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967104256:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775765262.022551:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765262.022568:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765262/real 1775765262] req@ffff8f994f2e4000 x1862022967104256/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765262.022583:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967104256/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765262.022594:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765262.022598:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765262.022602:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967104256:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775765262.022634:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967104384:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765262.022671:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765262.022680:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765262/real 1775765262] req@ffff8f994f2e7800 x1862022967104384/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765262.022690:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967104384/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765262.022696:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765262.022699:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765262.022702:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967104384:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765262.022724:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967104512:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765262.022789:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765262.022803:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765262/real 1775765262] req@ffff8f994f2e5180 x1862022967104512/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765262.022816:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967104512/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765262.022824:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765262.022827:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765262.022830:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967104512:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765262.917940:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765263.941970:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765264.965940:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765265.989928:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765267.013898:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765267.141894:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765267.141904:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765267.141909:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765267.141920:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1888 00000100:00080000:1.0:1775765267.141950:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765267.141959:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765267.142010:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967104640/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765267.142074:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765267.142077:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765267.142079:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765267.142125:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1888 00000100:00080000:1.0:1775765267.142131:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775765267.142136:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967104640:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775765267.142139:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765267.142152:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765267.142168:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967104768/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765267.142179:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765267.142183:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765267.142184:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765267.142188:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1888 00000100:00080000:1.0:1775765267.142224:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765267.142227:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765267.142231:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775765267.142235:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765267.142245:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967104896/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765267.142254:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765267.142257:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765267.142258:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765267.142261:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1888 00000100:00080000:1.0:1775765267.142264:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765267.142267:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765267.142270:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765267.142277:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765267/real 1775765267] req@ffff8f994f2e4a80 x1862022967104640/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765267.142281:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967105024/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765267.142298:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967104640/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765267.142311:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765267.142315:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765267.142319:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967104640:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765267.142340:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967104768:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775765267.142368:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765267.142378:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765267/real 1775765267] req@ffff8f994f2e6a00 x1862022967104768/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765267.142406:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967104768/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765267.142412:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765267.142414:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765267.142417:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967104768:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775765267.142421:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967104896:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765267.142435:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765267.142442:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765267/real 1775765267] req@ffff8f994f2e5f80 x1862022967104896/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765267.142450:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967104896/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765267.142455:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765267.142457:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765267.142459:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967104896:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765267.142463:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967105024:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765267.142474:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765267.142479:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765267/real 1775765267] req@ffff8f994f2e6680 x1862022967105024/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765267.142487:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967105024/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765267.142509:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765267.142511:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765267.142514:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967105024:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775765268.037945:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765269.061846:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765270.085874:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765271.109893:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765272.134026:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765272.262205:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765272.262221:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765272.262228:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765272.262244:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1893 00000100:00080000:0.0:1775765272.262260:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765272.262271:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765272.262327:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967105152/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765272.262392:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765272.262401:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765272.262405:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765272.262413:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1893 00000100:00080000:0.0:1775765272.262421:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765272.262427:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765272.262444:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765272.262468:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967105280/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765272.262488:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765272.262494:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1775765272.262509:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967105152:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775765272.262539:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765272.262551:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1893 00000100:00080000:0.0:1775765272.262558:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765272.262566:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765272.262576:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765272.262600:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967105408/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765272.262622:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775765272.262644:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765272.262651:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765272.262654:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765272.262662:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1893 00000100:00080000:0.0:1775765272.262689:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765272.262697:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765272.262723:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765272/real 1775765272] req@ffff8f9970a77480 x1862022967105152/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1775765272.262787:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765272.262817:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967105536/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765272.262856:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967105152/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765272.262881:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765272.262890:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765272.262897:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967105152:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765272.262910:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967105280:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765272.262958:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765272.263028:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765272/real 1775765272] req@ffff8f9970a77800 x1862022967105280/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765272.263064:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967105280/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765272.263082:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765272.263089:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765272.263095:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967105280:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765272.263307:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967105408:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765272.263360:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765272.263389:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765272/real 1775765272] req@ffff8f9970a76a00 x1862022967105408/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765272.263411:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967105408/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765272.263425:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765272.263431:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765272.263437:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967105408:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765272.263449:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967105536:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765272.263486:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765272.263507:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765272/real 1775765272] req@ffff8f9970a76d80 x1862022967105536/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765272.263528:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967105536/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765272.263542:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765272.263548:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765272.263554:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967105536:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775765273.157915:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765274.182023:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765275.205993:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765276.229985:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765277.254000:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765277.381870:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765277.381879:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765277.381882:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765277.381891:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1898 00000100:00080000:3.0:1775765277.381900:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765277.381906:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765277.381956:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967105664/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765277.381979:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765277.381982:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765277.381984:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765277.381987:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1898 00000100:00080000:3.0:1775765277.381991:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765277.381994:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765277.382004:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765277.382013:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967105664:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765277.382015:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967105792/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765277.382023:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765277.382025:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765277.382026:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765277.382029:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1898 00000100:00080000:3.0:1775765277.382031:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765277.382033:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765277.382035:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765277.382045:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967105920/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765277.382051:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765277.382053:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765277.382055:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765277.382057:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1898 00000100:00080000:3.0:1775765277.382059:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765277.382061:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765277.382063:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765277.382071:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967106048/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765277.382096:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765277.382136:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765277/real 1775765277] req@ffff8f9970ede300 x1862022967105664/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765277.382150:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967105664/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765277.382157:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765277.382161:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765277.382164:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967105664:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765277.382184:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967105792:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765277.382199:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765277.382206:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765277/real 1775765277] req@ffff8f9970edf800 x1862022967105792/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765277.382214:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967105792/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765277.382219:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765277.382221:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765277.382223:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967105792:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765277.382227:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967105920:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765277.382237:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765277.382242:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765277/real 1775765277] req@ffff8f9970ede680 x1862022967105920/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765277.382266:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967105920/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765277.382272:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765277.382274:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765277.382276:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967105920:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765277.382279:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967106048:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765277.382291:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765277.382297:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765277/real 1775765277] req@ffff8f9970edc700 x1862022967106048/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765277.382304:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967106048/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765277.382309:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765277.382310:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765277.382312:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967106048:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765278.277914:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765279.302022:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765280.326058:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765281.349860:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765282.373893:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765282.501860:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765282.501872:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765282.501921:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765282.501933:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1904 00000100:00080000:3.0:1775765282.501944:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765282.501951:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765282.502001:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967106176/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765282.502028:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765282.502032:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765282.502035:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765282.502040:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1904 00000100:00080000:3.0:1775765282.502044:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765282.502049:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765282.502059:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765282.502068:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967106176:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765282.502073:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967106304/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765282.502085:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765282.502089:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765282.502091:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765282.502095:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1904 00000100:00080000:3.0:1775765282.502098:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765282.502102:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765282.502107:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765282.502117:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967106432/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765282.502133:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765282.502137:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:2.0:1775765282.502138:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775765282.502139:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765282.502143:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1904 00000100:00080000:3.0:1775765282.502146:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765282.502149:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765282.502165:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765282.502175:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967106560/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765282.502206:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765282/real 1775765282] req@ffff8f9970eded80 x1862022967106176/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765282.502286:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967106176/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765282.502295:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765282.502298:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765282.502301:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967106176:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765282.502321:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967106304:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765282.502342:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765282.502353:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765282/real 1775765282] req@ffff8f9970edc000 x1862022967106304/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765282.502362:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967106304/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765282.502368:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765282.502370:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765282.502372:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967106304:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765282.502376:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967106432:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765282.502388:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765282.502395:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765282/real 1775765282] req@ffff8f9970edce00 x1862022967106432/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765282.502403:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967106432/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765282.502428:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765282.502431:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765282.502433:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967106432:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765282.502438:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967106560:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765282.502451:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765282.502458:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765282/real 1775765282] req@ffff8f9970edd500 x1862022967106560/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765282.502467:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967106560/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765282.502473:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765282.502475:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765282.502477:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967106560:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765283.397936:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765284.421906:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765285.445976:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765286.470012:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765287.494020:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765287.621905:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765287.621919:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765287.621924:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765287.621937:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1909 00000100:00080000:1.0:1775765287.621954:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765287.621962:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765287.622016:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967106688/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765287.622053:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765287.622057:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765287.622059:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765287.622063:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1909 00000100:00080000:1.0:1775765287.622066:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765287.622070:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765287.622081:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765287.622095:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967106816/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765287.622116:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775765287.622127:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967106688:192.168.202.146@tcp:250:kworker.0 00000800:00000100:2.0:1775765287.622212:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765287.622254:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765287/real 1775765287] req@ffff8f994f2e7b80 x1862022967106688/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765287.622270:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765287.622276:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765287.622280:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967106688/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765287.622287:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1909 00000100:00080000:1.0:1775765287.622315:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765287.622323:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765287.622335:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1775765287.622371:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765287.622377:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765287.622382:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967106944/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765287.622386:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967106688:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765287.622413:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967106816:192.168.202.146@tcp:38:kworker.0 00000100:00080000:1.0:1775765287.622415:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765287.622422:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765287.622423:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765287.622428:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1909 00000100:00080000:1.0:1775765287.622433:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765287.622437:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765287.622459:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765287.622471:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765287.622472:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967107072/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765287.622488:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765287/real 1775765287] req@ffff8f994f2e6300 x1862022967106816/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765287.622499:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967106816/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765287.622505:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765287.622508:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765287.622512:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967106816:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765287.622522:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967106944:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765287.622536:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765287.622543:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765287/real 1775765287] req@ffff8f994f2e4e00 x1862022967106944/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765287.622556:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967106944/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765287.622566:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765287.622570:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765287.622574:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967106944:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765287.622599:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967107072:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765287.622640:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765287.622655:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765287/real 1775765287] req@ffff8f994f2e4700 x1862022967107072/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765287.622671:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967107072/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765287.622681:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765287.622684:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765287.622687:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967107072:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765288.518009:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765289.541966:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765290.565931:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765291.589976:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765292.613992:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765292.741978:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765292.741998:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765292.742006:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765292.742024:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1914 00000100:00080000:1.0:1775765292.742047:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765292.742240:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765292.742280:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7100 x1862022967107200/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765292.742318:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765292.742323:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765292.742326:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765292.742332:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1914 00000100:00080000:1.0:1775765292.742344:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765292.742348:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765292.742360:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765292.742374:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967107200:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775765292.742377:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967107328/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765292.742395:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765292.742399:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765292.742401:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765292.742406:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1914 00000100:00080000:1.0:1775765292.742410:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765292.742414:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765292.742420:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765292.742432:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967107456/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765292.742441:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765292.742446:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:2.0:1775765292.742464:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775765292.742482:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765292.742491:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1914 00000100:00080000:1.0:1775765292.742499:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765292.742505:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765292.742515:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765292.742529:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765292/real 1775765292] req@ffff8f994f2e7100 x1862022967107200/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765292.742544:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967107584/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765292.742561:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7100 x1862022967107200/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765292.742586:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765292.742596:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765292.742608:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967107200:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765292.742647:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967107328:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765292.742708:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765292.742725:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765292/real 1775765292] req@ffff8f994f2e4700 x1862022967107328/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765292.742772:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967107328/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765292.742801:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765292.742811:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765292.742817:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967107328:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765292.742830:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967107456:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765292.742857:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765292.742870:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765292/real 1775765292] req@ffff8f994f2e4e00 x1862022967107456/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765292.742881:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967107456/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765292.742888:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765292.742892:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765292.742894:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967107456:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765292.742899:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967107584:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765292.742914:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765292.742921:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765292/real 1775765292] req@ffff8f994f2e6300 x1862022967107584/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765292.742959:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967107584/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765292.742967:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765292.742970:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765292.742972:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967107584:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765293.638001:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765294.662024:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765295.685979:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765296.710108:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765297.733925:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765297.861857:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765297.861869:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765297.861874:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765297.861883:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1919 00000100:00080000:2.0:1775765297.861893:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765297.861900:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765297.861951:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0380 x1862022967107712/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765297.861984:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765297.861987:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765297.861989:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765297.861993:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1919 00000100:00080000:2.0:1775765297.861997:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765297.862000:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765297.862047:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967107712:192.168.202.146@tcp:250:kworker.0 00000008:00100000:2.0:1775765297.862055:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765297.862070:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0000 x1862022967107840/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765297.862084:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765297.862087:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765297.862089:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765297.862092:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1919 00000100:00080000:2.0:1775765297.862095:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765297.862099:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765297.862103:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765297.862116:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2680 x1862022967107968/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765297.862125:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765297.862130:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765297.862131:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765297.862134:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1919 00000100:00080000:2.0:1775765297.862137:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765297.862143:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:0.0:1775765297.862150:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:2.0:1775765297.862154:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765297.862169:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3800 x1862022967108096/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765297.862211:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765297/real 1775765297] req@ffff8f99703d0380 x1862022967107712/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765297.862229:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0380 x1862022967107712/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765297.862239:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765297.862342:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765297.862349:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967107712:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765297.862381:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967107840:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775765297.862440:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765297.862463:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765297/real 1775765297] req@ffff8f99703d0000 x1862022967107840/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765297.862481:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0000 x1862022967107840/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765297.862491:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765297.862494:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765297.862498:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967107840:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775765297.862503:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967107968:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765297.862518:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765297.862527:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765297/real 1775765297] req@ffff8f99703d2680 x1862022967107968/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765297.862538:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2680 x1862022967107968/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765297.862545:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765297.862548:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765297.862551:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967107968:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765297.862556:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967108096:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765297.862593:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765297.862602:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765297/real 1775765297] req@ffff8f99703d3800 x1862022967108096/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765297.862613:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3800 x1862022967108096/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765297.862620:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765297.862623:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765297.862626:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967108096:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775765298.757940:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765299.781875:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765300.805911:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765301.829982:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765302.853911:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765302.981902:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765302.981913:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765302.981919:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765302.981934:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1924 00000100:00080000:0.0:1775765302.981949:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765302.981962:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765302.982022:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967108224/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765302.982070:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765302.982076:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765302.982079:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765302.982085:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1924 00000100:00080000:0.0:1775765302.982092:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765302.982097:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765302.982111:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765302.982157:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967108224:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765302.982174:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967108352/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765302.982195:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765302.982200:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765302.982202:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765302.982207:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1924 00000100:00080000:0.0:1775765302.982213:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765302.982218:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765302.982225:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765302.982257:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967108480/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765302.982267:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765302.982271:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765302.982273:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765302.982276:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1924 00000100:00080000:0.0:1775765302.982280:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765302.982283:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765302.982290:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765302.982294:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765302.982307:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967108608/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765302.982508:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765302/real 1775765302] req@ffff8f9970a74380 x1862022967108224/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765302.982541:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967108224/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765302.982558:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765302.982565:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765302.982571:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967108224:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765302.982620:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967108352:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765302.982662:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765302.982683:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765302/real 1775765302] req@ffff8f9970a75c00 x1862022967108352/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765302.982724:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967108352/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765302.982774:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765302.982782:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765302.982787:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967108352:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765302.982797:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967108480:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765302.982835:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765302.982899:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765302/real 1775765302] req@ffff8f9970a74700 x1862022967108480/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765302.982921:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967108480/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765302.982934:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765302.982939:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765302.982945:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967108480:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765302.982955:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967108608:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765302.982985:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765302.982998:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765302/real 1775765302] req@ffff8f9970a76d80 x1862022967108608/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765302.983053:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967108608/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765302.983062:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765302.983066:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765302.983069:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967108608:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775765303.877896:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765304.901895:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765305.925903:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765306.949909:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765307.973966:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765308.101973:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765308.101985:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765308.101991:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765308.102004:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1929 00000100:00080000:1.0:1775765308.102020:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765308.102031:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765308.102081:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967108736/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765308.102111:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765308.102115:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765308.102118:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765308.102123:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1929 00000100:00080000:1.0:1775765308.102127:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765308.102132:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765308.102193:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967108736:192.168.202.146@tcp:250:kworker.0 00000800:00000100:2.0:1775765308.102310:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:1.0:1775765308.102320:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765308.102381:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967108864/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765308.102390:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765308/real 1775765308] req@ffff8f994f2e7b80 x1862022967108736/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765308.102426:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765308.102431:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765308.102435:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765308.102442:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1929 00000100:00080000:1.0:1775765308.102451:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765308.102460:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765308.102473:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765308.102516:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967108992/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765308.102516:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967108736/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765308.102568:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765308.102585:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00080000:1.0:1775765308.102595:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775765308.102595:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967108736:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775765308.102609:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765308.102612:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765308.102619:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1929 00000100:00080000:1.0:1775765308.102624:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765308.102628:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765308.102636:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765308.102639:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967108864:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775765308.102680:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967109120/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765308.102718:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765308.102771:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765308/real 1775765308] req@ffff8f994f2e6680 x1862022967108864/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765308.102789:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967108864/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765308.102799:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765308.102802:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765308.102806:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967108864:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765308.102813:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967108992:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765308.102833:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765308.102853:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765308/real 1775765308] req@ffff8f994f2e5f80 x1862022967108992/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765308.102865:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967108992/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765308.102901:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765308.102905:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765308.102909:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967108992:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765308.102927:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967109120:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765308.102946:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765308.102957:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765308/real 1775765308] req@ffff8f994f2e6a00 x1862022967109120/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765308.102987:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967109120/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765308.102996:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765308.103000:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765308.103003:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967109120:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765308.997938:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765310.021905:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765311.045907:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765312.069944:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765313.093992:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765313.221854:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765313.221867:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765313.221873:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765313.221885:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1934 00000100:00080000:1.0:1775765313.221899:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765313.221910:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765313.221986:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967109248/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765313.222027:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765313.222032:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765313.222035:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765313.222045:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1934 00000100:00080000:1.0:1775765313.222055:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765313.222064:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765313.222079:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765313.222090:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967109248:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775765313.222114:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967109376/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765313.222152:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765313.222175:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765313.222177:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765313.222185:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1934 00000100:00080000:1.0:1775765313.222206:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765313.222212:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765313.222220:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765313.222288:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967109504/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765313.222313:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765313.222318:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765313.222323:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765313.222333:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1934 00000100:00080000:1.0:1775765313.222339:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765313.222345:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765313.222354:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765313.222354:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967109376:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775765313.222389:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967109632/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765313.222410:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967109504:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765313.222442:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967109632:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765314.117936:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765315.141927:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765316.166055:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775765316.293977:0:2366:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:3.0:1775765316.294015:0:2366:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775765316.294031:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:3.0:1775765316.294075:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:3.0:1775765316.294092:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:3.0:1775765316.294101:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:2.0:1775765316.294181:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765313/real 1775765316] req@ffff8f994f2e4000 x1862022967109632/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765316.294211:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967109632/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765316.294222:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765316.294228:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765316.294232:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967109632:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765316.294241:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765313/real 1775765316] req@ffff8f994f2e5180 x1862022967109376/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765316.294253:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967109376/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765316.294261:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765316.294265:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765316.294269:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967109376:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765316.294275:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765313/real 1775765316] req@ffff8f994f2e7800 x1862022967109504/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765316.294287:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967109504/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765316.294293:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765316.294296:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765316.294299:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967109504:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765316.294351:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765313/real 1775765316] req@ffff8f994f2e4a80 x1862022967109248/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765316.294362:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967109248/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765316.294370:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765316.294373:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765316.294376:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967109248:192.168.202.146@tcp:250:kworker.0 00010000:00010000:1.0:1775765317.189907:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765318.213960:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765318.341909:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765318.341921:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765318.341929:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765318.341952:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1939 00000100:00080000:1.0:1775765318.341974:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765318.341985:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765318.342031:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967109760/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765318.342061:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765318.342064:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765318.342065:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765318.342121:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967109760:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775765318.342188:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1939 00000100:00080000:1.0:1775765318.342198:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775765318.342204:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775765318.342209:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765318.342224:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765318.342283:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967109888/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765318.342292:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765318/real 1775765318] req@ffff8f994f2e6d80 x1862022967109760/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765318.342359:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967109760/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765318.342365:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765318.342376:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765318.342380:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765318.342401:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765318.342406:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1939 00000100:00080000:2.0:1775765318.342414:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00080000:1.0:1775765318.342416:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775765318.342425:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967109760:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775765318.342428:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765318.342445:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765318.342470:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967110016/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765318.342477:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967109888:192.168.202.146@tcp:38:kworker.0 00000100:00080000:1.0:1775765318.342503:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765318.342509:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765318.342514:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765318.342522:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1939 00000800:00000100:2.0:1775765318.342530:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775765318.342531:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765318.342540:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765318.342578:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765318/real 1775765318] req@ffff8f994f2e4a80 x1862022967109888/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1775765318.342594:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765318.342599:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967109888/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765318.342623:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967110144/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765318.342651:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765318.342664:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765318.342676:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967109888:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765318.342706:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967110016:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765318.342781:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765318.342802:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765318/real 1775765318] req@ffff8f994f2e7800 x1862022967110016/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765318.342821:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967110016/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765318.342833:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765318.342839:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765318.342844:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967110016:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765318.342853:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967110144:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765318.342885:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765318.342902:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765318/real 1775765318] req@ffff8f994f2e5180 x1862022967110144/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765318.342920:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967110144/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765318.342932:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765318.342936:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765318.342967:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967110144:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765319.237870:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765320.261910:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765321.285916:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765322.309949:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765323.333868:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765323.461849:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765323.461860:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765323.461864:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765323.461874:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1945 00000100:00080000:2.0:1775765323.461884:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765323.461891:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765323.461963:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1500 x1862022967110272/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765323.461999:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765323.462004:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765323.462006:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765323.462011:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1945 00000100:00080000:2.0:1775765323.462015:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765323.462019:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765323.462029:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765323.462043:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0e00 x1862022967110400/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765323.462055:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765323.462067:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765323.462070:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775765323.462124:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967110272:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775765323.462134:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1945 00000100:00080000:2.0:1775765323.462151:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765323.462156:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765323.462163:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765323.462190:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0700 x1862022967110528/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765323.462203:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765323.462206:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765323.462208:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765323.462213:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1945 00000100:00080000:2.0:1775765323.462216:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765323.462220:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765323.462226:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765323.462241:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2d80 x1862022967110656/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765323.462296:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765323.462387:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765323/real 1775765323] req@ffff8f99703d1500 x1862022967110272/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765323.462624:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1500 x1862022967110272/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765323.462636:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765323.462640:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765323.462645:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967110272:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765323.462656:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967110400:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775765323.462713:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765323.462735:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765323/real 1775765323] req@ffff8f99703d0e00 x1862022967110400/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765323.462786:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0e00 x1862022967110400/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765323.462794:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765323.462796:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765323.462799:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967110400:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775765323.462825:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967110528:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765323.462843:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765323.462852:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765323/real 1775765323] req@ffff8f99703d0700 x1862022967110528/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765323.462862:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0700 x1862022967110528/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765323.462913:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765323.462917:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765323.462921:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967110528:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765323.462928:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967110656:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765323.462952:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765323.462964:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765323/real 1775765323] req@ffff8f99703d2d80 x1862022967110656/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765323.462977:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2d80 x1862022967110656/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765323.462985:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765323.462988:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765323.462992:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967110656:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765324.357955:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765325.381929:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765326.406039:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765327.429854:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765328.453913:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765328.581827:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765328.581837:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765328.581841:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765328.581850:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1950 00000100:00080000:2.0:1775765328.581862:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765328.581868:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765328.581953:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0a80 x1862022967110784/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765328.581989:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765328.581994:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765328.581998:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765328.582003:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1950 00000100:00080000:2.0:1775765328.582008:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765328.582015:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765328.582026:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765328.582049:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967110784:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765328.582054:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1f80 x1862022967110912/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765328.582068:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765328.582072:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765328.582074:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765328.582078:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1950 00000100:00080000:2.0:1775765328.582083:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765328.582086:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765328.582091:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765328.582106:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1180 x1862022967111040/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765328.582112:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765328.582115:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765328.582116:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765328.582121:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1950 00000100:00080000:2.0:1775765328.582124:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765328.582127:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765328.582131:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775765328.582140:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765328.582143:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3100 x1862022967111168/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765328.582210:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765328/real 1775765328] req@ffff8f99703d0a80 x1862022967110784/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765328.582235:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0a80 x1862022967110784/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765328.582251:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765328.582263:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765328.582273:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967110784:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765328.582588:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967110912:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775765328.582679:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765328.582706:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765328/real 1775765328] req@ffff8f99703d1f80 x1862022967110912/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765328.582720:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1f80 x1862022967110912/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765328.582729:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765328.582732:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765328.582771:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967110912:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775765328.582780:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967111040:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775765328.582818:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765328.582846:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765328/real 1775765328] req@ffff8f99703d1180 x1862022967111040/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765328.582860:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1180 x1862022967111040/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765328.582869:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765328.582872:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765328.582876:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967111040:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775765328.582882:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967111168:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775765328.582901:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765328.582911:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765328/real 1775765328] req@ffff8f99703d3100 x1862022967111168/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765328.582923:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3100 x1862022967111168/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765328.582932:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765328.582935:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765328.582939:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967111168:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775765329.477884:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765330.501928:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765331.525993:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765332.549978:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765333.573923:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765333.701946:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765333.701958:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765333.701964:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765333.701976:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1955 00000100:00080000:3.0:1775765333.701991:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765333.701999:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765333.702049:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddc00 x1862022967111296/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765333.702105:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765333.702110:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765333.702113:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765333.702119:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1955 00000100:00080000:3.0:1775765333.702123:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765333.702127:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765333.702139:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765333.702167:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf100 x1862022967111424/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765333.702182:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765333.702189:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765333.702193:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775765333.702196:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967111296:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775765333.702200:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1955 00000100:00080000:3.0:1775765333.702208:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765333.702218:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765333.702224:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765333.702240:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967111552/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765333.702251:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765333.702256:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765333.702258:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765333.702272:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1955 00000100:00080000:3.0:1775765333.702277:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765333.702281:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765333.702287:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765333.702299:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967111680/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765333.702327:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765333.702464:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765333/real 1775765333] req@ffff8f9970eddc00 x1862022967111296/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765333.702489:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddc00 x1862022967111296/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765333.702512:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765333.702518:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765333.702523:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967111296:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765333.702595:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967111424:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775765333.702625:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765333.702642:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765333/real 1775765333] req@ffff8f9970edf100 x1862022967111424/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765333.702659:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf100 x1862022967111424/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765333.702669:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765333.702674:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765333.702678:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967111424:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775765333.702686:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967111552:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765333.702708:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765333.702720:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765333/real 1775765333] req@ffff8f9970edd500 x1862022967111552/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765333.702733:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967111552/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765333.702770:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765333.702775:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765333.702780:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967111552:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765333.702789:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967111680:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765333.702832:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765333.702855:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765333/real 1775765333] req@ffff8f9970edce00 x1862022967111680/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765333.702904:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967111680/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765333.702914:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765333.702918:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765333.702923:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967111680:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775765334.597894:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765335.622020:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765336.645907:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765337.670044:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765338.694160:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765338.822698:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765338.822716:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765338.822723:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765338.822775:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1960 00000100:00080000:3.0:1775765338.822795:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765338.822806:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765338.822865:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967111808/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765338.822908:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765338.822914:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765338.822916:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765338.822923:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1960 00000100:00080000:3.0:1775765338.822937:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765338.822943:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765338.822956:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765338.822972:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967111936/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765338.822981:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967111808:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775765338.822988:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765338.822993:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765338.822995:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765338.823000:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1960 00000100:00080000:3.0:1775765338.823003:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765338.823006:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765338.823011:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765338.823023:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967112064/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765338.823039:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765338.823042:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765338.823044:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765338.823047:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1960 00000100:00080000:3.0:1775765338.823051:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765338.823053:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765338.823057:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765338.823065:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967112192/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765338.823091:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765338.823153:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765338/real 1775765338] req@ffff8f9970edc000 x1862022967111808/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765338.823189:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967111808/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765338.823201:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765338.823206:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765338.823210:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967111808:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765338.823241:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967111936:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775765338.823266:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765338.823278:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765338/real 1775765338] req@ffff8f9970eded80 x1862022967111936/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765338.823300:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967111936/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765338.823536:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765338.823546:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765338.823555:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967111936:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775765338.823574:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967112064:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765338.823674:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765338.823704:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765338/real 1775765338] req@ffff8f9970edc700 x1862022967112064/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765338.823720:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967112064/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765338.823728:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765338.823731:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765338.823735:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967112064:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765338.823741:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967112192:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765338.823761:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765338.823815:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765338/real 1775765338] req@ffff8f9970ede680 x1862022967112192/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765338.823834:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967112192/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765338.823846:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765338.823851:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765338.823887:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967112192:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765339.717992:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765340.741923:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765341.766045:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765342.789943:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765343.813986:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765343.941907:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765343.941919:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765343.941924:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765343.941936:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1965 00000100:00080000:2.0:1775765343.941949:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765343.941956:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765343.942008:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3480 x1862022967112320/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765343.942042:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765343.942047:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765343.942050:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765343.942055:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1965 00000100:00080000:2.0:1775765343.942058:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765343.942062:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765343.942073:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765343.942090:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2300 x1862022967112448/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765343.942102:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765343.942106:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765343.942107:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765343.942112:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1965 00000100:00080000:2.0:1775765343.942117:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765343.942122:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765343.942129:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765343.942141:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967112320:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765343.942144:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1c00 x1862022967112576/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765343.942173:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765343.942177:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765343.942179:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765343.942184:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1965 00000100:00080000:2.0:1775765343.942188:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765343.942192:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765343.942197:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765343.942254:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3b80 x1862022967112704/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765343.942260:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765343.942353:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765343/real 1775765343] req@ffff8f99703d3480 x1862022967112320/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765343.942400:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3480 x1862022967112320/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765343.942413:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765343.942418:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765343.942422:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967112320:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765343.942431:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967112448:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775765343.942457:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765343.942469:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765343/real 1775765343] req@ffff8f99703d2300 x1862022967112448/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765343.942480:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2300 x1862022967112448/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765343.942487:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765343.942490:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765343.942493:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967112448:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775765343.942542:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967112576:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765343.942558:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765343.942837:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765343/real 1775765343] req@ffff8f99703d1c00 x1862022967112576/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765343.942862:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1c00 x1862022967112576/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765343.942874:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765343.942879:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765343.942884:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967112576:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765343.942895:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967112704:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765343.942952:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765343.942963:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765343/real 1775765343] req@ffff8f99703d3b80 x1862022967112704/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765343.942976:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3b80 x1862022967112704/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765343.942985:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765343.942989:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765343.942992:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967112704:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765344.837905:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765345.862421:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765346.886027:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765347.909954:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765348.933993:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765349.061948:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765349.061964:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765349.061984:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765349.062007:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1970 00000100:00080000:1.0:1775765349.062025:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765349.062035:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765349.062231:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967112832/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765349.062294:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765349.062300:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765349.062303:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765349.062329:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1970 00000100:00080000:1.0:1775765349.062343:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765349.062352:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765349.062364:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967112832:192.168.202.146@tcp:250:kworker.0 00000008:00100000:1.0:1775765349.062369:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765349.062407:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967112960/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765349.062419:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765349.062422:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765349.062424:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765349.062429:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1970 00000100:00080000:1.0:1775765349.062459:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765349.062464:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765349.062470:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775765349.062479:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765349.062491:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967113088/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765349.062505:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765349.062511:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765349.062513:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765349.062517:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1970 00000100:00080000:1.0:1775765349.062521:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765349.062523:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765349.062529:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765349.062546:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967113216/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765349.062558:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765349/real 1775765349] req@ffff8f994f2e4000 x1862022967112832/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765349.062589:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967112832/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765349.062769:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765349.062776:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765349.062782:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967112832:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765349.062817:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967112960:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775765349.062880:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765349.062904:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765349/real 1775765349] req@ffff8f994f2e6a00 x1862022967112960/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765349.062937:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967112960/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765349.062946:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765349.062949:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765349.062952:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967112960:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775765349.062959:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967113088:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765349.062978:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765349.062987:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765349/real 1775765349] req@ffff8f994f2e5f80 x1862022967113088/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765349.062997:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967113088/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765349.063021:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765349.063023:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765349.063026:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967113088:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765349.063053:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967113216:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765349.063071:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765349.063080:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765349/real 1775765349] req@ffff8f994f2e6680 x1862022967113216/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765349.063090:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967113216/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765349.063097:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765349.063100:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765349.063102:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967113216:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765349.958058:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765350.981911:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765352.005992:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765353.030010:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765354.053962:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765354.181883:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765354.181896:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765354.181977:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765354.181989:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1975 00000100:00080000:1.0:1775765354.182004:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765354.182027:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765354.182110:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967113344/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765354.182149:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765354.182153:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765354.182155:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765354.182159:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1975 00000100:00080000:1.0:1775765354.182163:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765354.182166:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765354.182206:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765354.182264:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967113472/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765354.182280:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967113344:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775765354.182287:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765354.182291:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765354.182294:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765354.182301:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1975 00000100:00080000:1.0:1775765354.182305:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765354.182310:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765354.182318:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765354.182343:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967113600/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765354.182364:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765354.182367:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765354.182369:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765354.182373:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1975 00000100:00080000:1.0:1775765354.182378:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765354.182381:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765354.182385:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765354.182395:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967113728/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765354.182463:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765354.182531:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765354/real 1775765354] req@ffff8f994f2e7b80 x1862022967113344/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765354.182590:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967113344/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765354.182605:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765354.182610:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765354.182625:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967113344:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765354.182674:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967113472:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775765354.182705:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765354.182721:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765354/real 1775765354] req@ffff8f994f2e6300 x1862022967113472/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765354.182801:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967113472/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765354.182835:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765354.182839:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765354.182843:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967113472:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775765354.182852:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967113600:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765354.182894:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765354.182913:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765354/real 1775765354] req@ffff8f994f2e4e00 x1862022967113600/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765354.182935:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967113600/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765354.182949:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765354.182956:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765354.182962:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967113600:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765354.182974:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967113728:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765354.183010:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765354.183031:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765354/real 1775765354] req@ffff8f994f2e4700 x1862022967113728/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765354.183064:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967113728/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765354.183107:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765354.183112:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765354.183116:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967113728:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765355.078051:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765356.101890:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765357.125978:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765358.149947:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765359.174043:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765359.301859:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765359.301867:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765359.301871:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765359.301880:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1980 00000100:00080000:3.0:1775765359.301891:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765359.301897:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765359.301948:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967113856/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765359.301975:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765359.301978:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765359.301979:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765359.301983:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1980 00000100:00080000:3.0:1775765359.301986:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765359.301988:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765359.302007:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765359.302015:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967113856:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765359.302025:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967113984/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765359.302034:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765359.302036:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765359.302037:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765359.302040:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1980 00000100:00080000:3.0:1775765359.302043:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765359.302046:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765359.302050:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765359.302057:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967114112/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765359.302064:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765359.302066:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765359.302068:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765359.302072:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1980 00000800:00000100:0.0:1775765359.302076:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775765359.302081:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765359.302087:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765359.302093:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765359.302109:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765359/real 1775765359] req@ffff8f9970edf800 x1862022967113856/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765359.302122:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967114240/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765359.302128:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967113856/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765359.302147:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765359.302152:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765359.302156:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967113856:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765359.302198:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967113984:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775765359.302234:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765359.302245:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765359/real 1775765359] req@ffff8f9970ede300 x1862022967113984/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765359.302255:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967113984/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765359.302262:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765359.302264:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765359.302267:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967113984:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775765359.302296:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967114112:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765359.302321:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765359.302334:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765359/real 1775765359] req@ffff8f9970edf480 x1862022967114112/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765359.302349:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967114112/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765359.302359:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765359.302362:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765359.302366:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967114112:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765359.302371:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967114240:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765359.302389:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765359.302397:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765359/real 1775765359] req@ffff8f9970edd880 x1862022967114240/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765359.302406:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967114240/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765359.302431:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765359.302433:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765359.302435:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967114240:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775765360.197897:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765361.221916:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765362.245939:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765363.269850:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765364.293901:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765364.421876:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765364.421889:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765364.421898:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765364.421913:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1986 00000100:00080000:3.0:1775765364.421930:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765364.421940:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765364.421988:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967114368/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765364.422023:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765364.422029:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765364.422032:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765364.422038:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1986 00000100:00080000:3.0:1775765364.422044:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765364.422049:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765364.422062:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765364.422073:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967114368:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765364.422085:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967114496/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765364.422116:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765364.422131:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765364.422133:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765364.422137:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1986 00000100:00080000:3.0:1775765364.422141:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765364.422144:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765364.422149:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765364.422177:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967114624/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765364.422190:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765364.422193:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765364.422194:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765364.422197:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1986 00000100:00080000:3.0:1775765364.422199:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765364.422201:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765364.422204:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765364.422213:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967114752/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765364.422220:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765364.422253:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765364/real 1775765364] req@ffff8f9970edfb80 x1862022967114368/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765364.422269:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967114368/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765364.422285:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765364.422289:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765364.422293:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967114368:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775765364.422315:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967114496:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775765364.422338:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765364.422368:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765364/real 1775765364] req@ffff8f9970edea00 x1862022967114496/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765364.422384:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967114496/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765364.422393:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765364.422398:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765364.422403:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967114496:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775765364.422410:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967114624:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765364.422447:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765364.422457:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765364/real 1775765364] req@ffff8f9970edca80 x1862022967114624/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765364.422467:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967114624/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765364.422474:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765364.422478:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765364.422482:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967114624:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775765364.422506:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967114752:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765364.422553:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765364.422582:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765364/real 1775765364] req@ffff8f9970edc380 x1862022967114752/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765364.422594:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967114752/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765364.422611:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765364.422614:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765364.422616:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967114752:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775765365.317905:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765366.342019:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765367.365958:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765368.389929:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765369.413970:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765369.541822:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765369.541830:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765369.541835:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765369.541843:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1991 00000100:00080000:3.0:1775765369.541859:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765369.541864:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765369.541895:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddf80 x1862022967114880/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765369.541916:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765369.541919:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765369.541920:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765369.541923:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1991 00000100:00080000:3.0:1775765369.541926:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765369.541928:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765369.541936:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765369.541944:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd180 x1862022967115008/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765369.541954:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765369.541957:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765369.541958:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765369.541960:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1991 00000100:00080000:3.0:1775765369.541962:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765369.541965:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765369.541968:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765369.541976:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967115136/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765369.541984:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765369.541987:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765369.541988:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765369.541992:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1991 00000100:00100000:1.0:1775765369.541994:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967114880:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775765369.541995:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765369.541999:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765369.542005:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765369.542012:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967115264/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765369.542058:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765369.542108:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765369/real 1775765369] req@ffff8f9970eddf80 x1862022967114880/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765369.542122:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddf80 x1862022967114880/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765369.542130:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765369.542133:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765369.542136:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967114880:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775765369.542141:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967115008:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775765369.542155:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765369.542163:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765369/real 1775765369] req@ffff8f9970edd180 x1862022967115008/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765369.542170:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd180 x1862022967115008/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765369.542175:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765369.542221:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765369.542225:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967115008:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775765369.542249:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967115136:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765369.542272:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765369.542284:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765369/real 1775765369] req@ffff8f9970edc380 x1862022967115136/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765369.542297:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967115136/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765369.542305:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765369.542319:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765369.542323:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967115136:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775765369.542333:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967115264:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765369.542354:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765369.542366:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765369/real 1775765369] req@ffff8f9970edca80 x1862022967115264/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765369.542378:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967115264/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765369.542386:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765369.542389:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765369.542393:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967115264:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765370.437862:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765371.461900:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765372.486012:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765373.509996:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765374.534179:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765374.661931:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765374.662157:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765374.662162:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765374.662175:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 1996 00000100:00080000:0.0:1775765374.662186:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765374.662210:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765374.662262:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967115392/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765374.662290:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765374.662293:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765374.662294:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765374.662307:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1996 00000100:00080000:0.0:1775765374.662310:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765374.662313:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765374.662322:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765374.662338:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967115520/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765374.662346:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765374.662348:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765374.662349:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765374.662351:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1996 00000100:00080000:0.0:1775765374.662354:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765374.662357:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765374.662362:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765374.662370:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967115648/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765374.662392:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765374.662394:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765374.662397:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765374.662400:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 1996 00000100:00080000:0.0:1775765374.662404:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765374.662412:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765374.662418:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967115392:192.168.202.146@tcp:250:kworker.0 00000008:00100000:0.0:1775765374.662419:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765374.662437:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967115776/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765374.662676:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765374.662795:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765374/real 1775765374] req@ffff8f9970a76a00 x1862022967115392/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765374.662824:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967115392/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765374.662840:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765374.662846:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765374.662852:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967115392:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775765374.662863:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967115520:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775765374.662918:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765374.662940:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765374/real 1775765374] req@ffff8f9970a77800 x1862022967115520/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765374.662965:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967115520/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765374.662982:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765374.662989:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765374.662996:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967115520:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775765374.663072:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967115648:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765374.663106:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765374.663122:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765374/real 1775765374] req@ffff8f9970a77480 x1862022967115648/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765374.663166:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967115648/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765374.663177:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765374.663181:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765374.663185:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967115648:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775765374.663193:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967115776:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765374.663216:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765374.663228:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765374/real 1775765374] req@ffff8f9970a75180 x1862022967115776/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765374.663242:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967115776/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765374.663252:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765374.663255:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765374.663259:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967115776:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765375.558149:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765376.581870:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765377.606108:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765378.629904:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765379.653959:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765379.781860:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765379.781871:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765379.781875:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765379.781887:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2001 00000100:00080000:0.0:1775765379.781900:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765379.781908:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765379.781949:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967115904/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765379.782004:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765379.782008:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765379.782010:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765379.782015:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2001 00000100:00080000:0.0:1775765379.782019:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765379.782032:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765379.782076:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967115904:192.168.202.146@tcp:250:kworker.0 00000008:00100000:0.0:1775765379.782085:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765379.782106:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967116032/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765379.782119:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765379.782122:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765379.782124:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765379.782128:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2001 00000100:00080000:0.0:1775765379.782140:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765379.782145:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765379.782150:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765379.782161:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967116160/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765379.782170:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765379.782175:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765379.782177:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765379.782181:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2001 00000100:00080000:0.0:1775765379.782184:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765379.782188:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765379.782193:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765379.782203:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967116032:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775765379.782208:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967116288/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765379.782295:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967116160:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775765379.782356:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967116288:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765380.678167:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765381.702011:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765382.726076:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775765382.854212:0:2368:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:3.0:1775765382.854276:0:2368:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775765382.854299:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:3.0:1775765382.854354:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:3.0:1775765382.854384:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:3.0:1775765382.854402:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:1.0:1775765382.854466:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765379/real 1775765382] req@ffff8f9970a75180 x1862022967116288/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765382.854548:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967116288/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765382.854571:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765382.854580:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765382.854588:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967116288:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775765382.854602:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765379/real 1775765382] req@ffff8f9970a76300 x1862022967116032/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765382.854626:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967116032/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765382.854642:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765382.854649:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765382.854656:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967116032:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775765382.854669:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765379/real 1775765382] req@ffff8f9970a77100 x1862022967116160/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765382.854691:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967116160/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765382.854705:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765382.854711:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765382.854717:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967116160:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775765382.854842:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765379/real 1775765382] req@ffff8f9970a75500 x1862022967115904/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765382.854871:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967115904/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765382.854887:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765382.854894:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765382.854900:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967115904:192.168.202.146@tcp:250:kworker.0 00010000:00010000:1.0:1775765383.749912:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765384.773906:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765384.901893:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765384.901902:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765384.901936:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765384.901947:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2006 00000100:00080000:3.0:1775765384.901973:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765384.901979:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765384.902048:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967116416/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765384.902083:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765384.902087:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765384.902089:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765384.902095:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2006 00000100:00080000:3.0:1775765384.902100:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765384.902103:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765384.902112:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765384.902194:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967116544/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765384.902209:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775765384.902213:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967116416:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775765384.902214:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765384.902216:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765384.902227:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2006 00000100:00080000:3.0:1775765384.902232:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765384.902236:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765384.902242:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765384.902272:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967116672/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765384.902283:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765384.902287:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765384.902289:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765384.902294:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2006 00000100:00080000:3.0:1775765384.902299:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765384.902303:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765384.902309:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775765384.902311:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765384.902327:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967116800/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765384.902388:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765384/real 1775765384] req@ffff8f9970edea00 x1862022967116416/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765384.902411:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967116416/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765384.902422:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765384.902427:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765384.902431:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967116416:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775765384.902459:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967116544:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775765384.902484:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765384.902498:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765384/real 1775765384] req@ffff8f9970edfb80 x1862022967116544/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765384.902548:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967116544/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765384.902556:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765384.902558:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765384.902560:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967116544:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775765384.902566:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967116672:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765384.902589:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765384.902601:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765384/real 1775765384] req@ffff8f9970edd880 x1862022967116672/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765384.902614:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967116672/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765384.902621:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765384.902625:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765384.902628:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967116672:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775765384.902634:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967116800:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765384.902655:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765384.902667:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765384/real 1775765384] req@ffff8f9970edf480 x1862022967116800/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765384.902679:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967116800/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765384.902711:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765384.902715:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765384.902719:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967116800:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765385.798005:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765386.821899:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765387.845884:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765388.870041:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765389.893972:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765390.021925:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765390.021937:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765390.021943:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765390.021956:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2011 00000100:00080000:2.0:1775765390.021971:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765390.021979:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765390.022027:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2a00 x1862022967116928/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765390.022055:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765390.022059:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765390.022060:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765390.022064:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2011 00000100:00080000:2.0:1775765390.022067:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765390.022069:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765390.022079:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765390.022089:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1880 x1862022967117056/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765390.022104:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765390.022108:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765390.022109:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765390.022117:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2011 00000100:00080000:2.0:1775765390.022121:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775765390.022125:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967116928:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775765390.022126:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:1.0:1775765390.022218:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:2.0:1775765390.022261:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765390.022288:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765390/real 1775765390] req@ffff8f99703d2a00 x1862022967116928/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765390.022321:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3b80 x1862022967117184/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765390.022323:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2a00 x1862022967116928/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765390.022378:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765390.022378:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765390.022393:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765390.022396:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765390.022398:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00080000:2.0:1775765390.022406:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2011 00000100:00100000:1.0:1775765390.022414:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967116928:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775765390.022421:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765390.022441:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765390.022446:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967117056:192.168.202.146@tcp:38:kworker.0 00000008:00100000:2.0:1775765390.022453:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765390.022481:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1c00 x1862022967117312/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765390.022515:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765390.022545:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765390/real 1775765390] req@ffff8f99703d1880 x1862022967117056/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765390.022572:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1880 x1862022967117056/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765390.022588:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765390.022594:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765390.022708:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967117056:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775765390.022833:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967117184:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765390.022935:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765390.022970:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765390/real 1775765390] req@ffff8f99703d3b80 x1862022967117184/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765390.022990:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3b80 x1862022967117184/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765390.023001:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765390.023005:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765390.023009:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967117184:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775765390.023015:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967117312:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765390.023038:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765390.023048:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765390/real 1775765390] req@ffff8f99703d1c00 x1862022967117312/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765390.023060:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1c00 x1862022967117312/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765390.023067:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765390.023070:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765390.023072:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967117312:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775765390.918053:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765391.941948:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765392.965926:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765393.990019:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765395.013986:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765395.141839:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765395.141848:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765395.141852:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765395.141861:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2016 00000100:00080000:2.0:1775765395.141871:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765395.141877:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765395.141907:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2300 x1862022967117440/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765395.141928:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765395.141952:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765395.141953:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765395.141957:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2016 00000100:00080000:2.0:1775765395.141960:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765395.141964:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765395.141972:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765395.141982:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3480 x1862022967117568/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765395.141990:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765395.141994:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765395.141996:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765395.142000:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2016 00000100:00100000:1.0:1775765395.142021:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967117440:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775765395.142027:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765395.142035:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765395.142041:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765395.142055:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3100 x1862022967117696/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765395.142064:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765395.142066:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765395.142068:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765395.142071:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2016 00000100:00080000:2.0:1775765395.142073:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765395.142076:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765395.142092:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765395.142100:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1180 x1862022967117824/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765395.142196:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765395.142244:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765395/real 1775765395] req@ffff8f99703d2300 x1862022967117440/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765395.142261:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2300 x1862022967117440/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765395.142270:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765395.142273:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765395.142276:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967117440:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775765395.142283:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967117568:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775765395.142297:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765395.142305:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765395/real 1775765395] req@ffff8f99703d3480 x1862022967117568/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765395.142312:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3480 x1862022967117568/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765395.142316:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765395.142318:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765395.142320:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967117568:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775765395.142342:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967117696:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765395.142353:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765395.142358:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765395/real 1775765395] req@ffff8f99703d3100 x1862022967117696/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765395.142399:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3100 x1862022967117696/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765395.142404:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765395.142406:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765395.142408:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967117696:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775765395.142412:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967117824:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765395.142424:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765395.142430:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765395/real 1775765395] req@ffff8f99703d1180 x1862022967117824/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765395.142437:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1180 x1862022967117824/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765395.142441:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765395.142443:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765395.142445:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967117824:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775765396.037978:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765397.061874:0:21596:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 3/3 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1775765397.061907:0:21596:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 3/6 locks to regular blp list, 2 blwis in pool 00010000:00010000:1.0:1775765397.061914:0:19800:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 3 locks of 3 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1775765397.061934:0:19800:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800028400000000 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:2.0:1775765397.061951:0:11210:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 3 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775765397.061959:0:11210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000587dd0e8/0x36f9bdc9e421ca0f lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x800028400000000 nid: local remote: 0x2aac6a50e5fc4ab expref: -99 pid: 22506 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1775765397.061979:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1775765397.062102:0:11210:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xee0:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775765397.062103:0:19800:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xee0:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775765397.062189:0:11210:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xee0:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:2.0:1775765397.062245:0:11210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800028400000000 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00000080:00200000:2.0:1775765397.062268:0:11210:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xee0:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775765397.062272:0:11210:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xee0:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:2.0:1775765397.062300:0:11210:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000004a200161/0x36f9bdc9e421c85d lrc: 2/0,0 mode: PW/PW res: [0x240000403:0x661:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x2aac6a50e5fb42e expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775765397.062328:0:11210:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xed9:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775765397.062365:0:11210:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xed9:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:2.0:1775765397.062430:0:11210:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000587dd0e8/0x36f9bdc9e421ca0f lrc: 1/0,0 mode: --/PW res: [0x240000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x804869400000020 nid: local remote: 0x2aac6a50e5fc4ab expref: -99 pid: 22506 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775765397.062438:0:11210:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 1/0,0 mode: --/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x804869400000020 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1775765397.062456:0:11210:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000004a200161/0x36f9bdc9e421c85d lrc: 1/0,0 mode: --/PW res: [0x240000403:0x661:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x2aac6a50e5fb42e expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775765397.062460:0:11210:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:2.0:1775765397.062485:0:11210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000587dd0e8/0x36f9bdc9e421ca0f lrc: 0/0,0 mode: --/PW res: [0x240000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x804869400000020 nid: local remote: 0x2aac6a50e5fc4ab expref: -99 pid: 22506 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775765397.062507:0:11210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 00000000d54dfb97/0x36f9bdc9e421c998 lrc: 0/0,0 mode: --/PW res: [0x240000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x804869400000020 nid: local remote: 0x2aac6a50e5fc2d6 expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1775765397.062521:0:11210:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8f99523ee800 lock: 000000004a200161/0x36f9bdc9e421c85d lrc: 0/0,0 mode: --/PW res: [0x240000403:0x661:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x2aac6a50e5fb42e expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00080000:3.0:1775765397.062530:0:2382:0:(client.c:1830:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != DISCONN) req@ffff8f99703d1f80 x1862022967117952/t0(0) o103->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:17/18 lens 336/224 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00010000:00010000:2.0:1775765397.062538:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765397.062543:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765397.062546:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1775765397.063241:0:19800:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xee0:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1775765397.063267:0:19800:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800028400000000 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1775765397.063300:0:19800:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xee0:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775765397.063304:0:19800:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xee0:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1775765397.063318:0:19800:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000030032e1e/0x36f9bdc9e421c856 lrc: 2/0,0 mode: PW/PW res: [0x280000403:0x660:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800028400000000 nid: local remote: 0x2aac6a50e5fb427 expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775765397.063331:0:19800:0:(vvp_io.c:1817:vvp_io_init()) [0x200000401:0xed9:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775765397.063333:0:19800:0:(vvp_io.c:288:vvp_io_fini()) [0x200000401:0xed9:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1775765397.063362:0:19800:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 1/0,0 mode: --/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x804869400000020 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775765397.063368:0:19800:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 1/0,0 mode: --/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x804869400000020 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775765397.063373:0:19800:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000030032e1e/0x36f9bdc9e421c856 lrc: 1/0,0 mode: --/PW res: [0x280000403:0x660:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x804869400000020 nid: local remote: 0x2aac6a50e5fb427 expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775765397.063377:0:19800:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:1.0:1775765397.063394:0:19800:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000031842f6/0x36f9bdc9e421c9c9 lrc: 0/0,0 mode: --/PW res: [0x280000403:0x704:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x804869400000020 nid: local remote: 0x2aac6a50e5fc393 expref: -99 pid: 22493 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775765397.063409:0:19800:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 00000000368402f4/0x36f9bdc9e421c991 lrc: 0/0,0 mode: --/PW res: [0x280000403:0x703:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x804869400000020 nid: local remote: 0x2aac6a50e5fc2cf expref: -99 pid: 22483 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775765397.063421:0:19800:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8f99523ee800 lock: 0000000030032e1e/0x36f9bdc9e421c856 lrc: 0/0,0 mode: --/PW res: [0x280000403:0x660:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x804869400000020 nid: local remote: 0x2aac6a50e5fb427 expref: -99 pid: 20529 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00080000:0.0:1775765397.063427:0:2380:0:(client.c:1830:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != DISCONN) req@ffff8f994f2e7100 x1862022967118080/t0(0) o103->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:17/18 lens 336/224 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00010000:00010000:1.0:1775765397.063452:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765397.063454:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765397.063456:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765398.086019:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765399.109953:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765400.133933:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765400.261840:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765400.261849:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765400.261853:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765400.261868:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2021 00000100:00080000:0.0:1775765400.261884:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765400.261892:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765400.261926:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967118208/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765400.261960:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765400.261964:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765400.261966:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765400.261971:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2021 00000100:00080000:0.0:1775765400.261975:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765400.261979:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765400.261989:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765400.262003:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967118336/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765400.262014:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967118208:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775765400.262017:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765400.262023:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765400.262025:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765400.262031:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2021 00000100:00080000:0.0:1775765400.262057:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765400.262063:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765400.262069:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765400.262084:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967118464/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765400.262096:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765400.262100:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765400.262102:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1775765400.262107:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775765400.262108:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2021 00000100:00080000:0.0:1775765400.262115:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765400.262120:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765400.262125:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765400.262137:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967118592/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765400.262146:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765400/real 1775765400] req@ffff8f9970a77480 x1862022967118208/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765400.262162:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967118208/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765400.262170:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765400.262184:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765400.262188:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967118208:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765400.262206:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967118336:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765400.262238:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765400.262247:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765400/real 1775765400] req@ffff8f9970a77800 x1862022967118336/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765400.262256:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967118336/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765400.262261:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765400.262264:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765400.262266:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967118336:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765400.262270:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967118464:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765400.262281:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765400.262287:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765400/real 1775765400] req@ffff8f9970a76a00 x1862022967118464/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765400.262294:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967118464/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765400.262300:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765400.262301:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765400.262303:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967118464:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765400.262307:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967118592:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765400.262330:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765400.262336:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765400/real 1775765400] req@ffff8f9970a76d80 x1862022967118592/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765400.262344:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967118592/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765400.262349:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765400.262351:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765400.262353:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967118592:192.168.202.146@tcp:8:kworker.0 00010000:00010000:0.0:1775765401.157924:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765402.181930:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765403.205919:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765404.229953:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765405.253918:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765405.381913:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765405.381926:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765405.381931:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765405.381944:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2026 00000100:00080000:0.0:1775765405.381958:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765405.381965:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765405.382046:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967118720/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765405.382107:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765405.382111:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765405.382113:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765405.382119:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2026 00000100:00080000:0.0:1775765405.382123:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765405.382137:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765405.382146:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967118720:192.168.202.146@tcp:250:kworker.0 00000008:00100000:0.0:1775765405.382148:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765405.382168:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967118848/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765405.382180:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765405.382184:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765405.382186:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765405.382190:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2026 00000800:00000100:1.0:1775765405.382209:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775765405.382216:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765405.382224:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765405.382231:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765405.382250:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967118976/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765405.382256:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765405/real 1775765405] req@ffff8f9970a74700 x1862022967118720/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765405.382264:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765405.382271:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765405.382275:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765405.382280:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2026 00000100:00100000:1.0:1775765405.382289:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967118720/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765405.382302:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765405.382304:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765405.382313:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00080000:1.0:1775765405.382314:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000008:00100000:0.0:1775765405.382322:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765405.382324:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967118720:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765405.382341:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967119104/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765405.382364:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967118848:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775765405.382394:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765405.382425:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765405/real 1775765405] req@ffff8f9970a75c00 x1862022967118848/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765405.382440:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967118848/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765405.382449:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765405.382458:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765405.382485:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967118848:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775765405.382491:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967118976:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765405.382509:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765405.382518:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765405/real 1775765405] req@ffff8f9970a74380 x1862022967118976/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765405.382527:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967118976/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765405.382533:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765405.382535:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765405.382537:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967118976:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775765405.382541:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967119104:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765405.382551:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765405.382557:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765405/real 1775765405] req@ffff8f9970a76d80 x1862022967119104/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765405.382565:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967119104/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765405.382570:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765405.382571:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765405.382573:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967119104:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775765406.278003:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765407.301886:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765408.325862:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765409.349841:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765410.373854:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765410.501788:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765410.501796:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765410.501799:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765410.501808:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2032 00000100:00080000:0.0:1775765410.501818:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765410.501823:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765410.501848:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967119232/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765410.501873:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765410.501876:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765410.501878:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765410.501881:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2032 00000100:00080000:0.0:1775765410.501884:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765410.501888:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765410.501896:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765410.501900:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967119232:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765410.501910:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967119360/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765410.501937:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765410.501940:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765410.501942:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765410.501945:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2032 00000100:00080000:0.0:1775765410.501949:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765410.501952:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765410.501956:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765410.501958:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765410.501970:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967119488/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765410.501978:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765410.501981:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765410.501983:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765410.501987:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2032 00000100:00100000:2.0:1775765410.501995:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765410/real 1775765410] req@ffff8f9970a76a00 x1862022967119232/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765410.502002:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765410.502008:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765410.502013:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765410.502016:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967119232/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765410.502025:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765410.502029:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765410.502032:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967119616/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765410.502035:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967119232:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765410.502061:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967119360:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765410.502097:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765410.502107:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765410/real 1775765410] req@ffff8f9970a77800 x1862022967119360/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765410.502116:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967119360/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765410.502122:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765410.502125:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765410.502127:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967119360:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765410.502132:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967119488:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765410.502144:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765410.502149:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765410/real 1775765410] req@ffff8f9970a77480 x1862022967119488/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765410.502157:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967119488/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765410.502162:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765410.502163:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765410.502165:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967119488:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765410.502169:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967119616:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765410.502178:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765410.502183:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765410/real 1775765410] req@ffff8f9970a75500 x1862022967119616/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765410.502207:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967119616/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765410.502212:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765410.502214:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765410.502216:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967119616:192.168.202.146@tcp:8:kworker.0 00010000:00010000:0.0:1775765411.397878:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765412.421938:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765413.445930:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765414.469877:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765415.493943:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765415.621859:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765415.621871:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765415.621877:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765415.621889:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2037 00000100:00080000:3.0:1775765415.621903:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765415.621911:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765415.621955:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967119744/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765415.621980:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765415.621983:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765415.621984:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765415.621988:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2037 00000100:00080000:3.0:1775765415.621991:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765415.621993:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765415.622003:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765415.622012:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967119872/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765415.622021:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765415.622024:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765415.622025:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765415.622030:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2037 00000100:00100000:2.0:1775765415.622032:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967119744:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775765415.622032:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765415.622036:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765415.622043:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765415.622050:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967120000/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765415.622056:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765415.622060:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765415.622061:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765415.622064:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2037 00000100:00080000:3.0:1775765415.622067:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765415.622069:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765415.622073:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765415.622113:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765415.622162:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765415/real 1775765415] req@ffff8f9970ede300 x1862022967119744/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765415.622180:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967119744/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765415.622189:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765415.622192:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765415.622195:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967119744:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765415.622217:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967119872:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765415.622233:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765415.622242:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765415/real 1775765415] req@ffff8f9970edf800 x1862022967119872/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765415.622249:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967119872/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765415.622298:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967120128/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765415.622395:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765415.622399:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765415.622402:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967119872:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765415.622407:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967120000:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765415.622426:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765415.622435:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765415/real 1775765415] req@ffff8f9970ede680 x1862022967120000/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765415.622442:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967120000/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765415.622447:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765415.622449:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765415.622451:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967120000:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765415.622461:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967120128:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765415.622472:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765415.622478:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765415/real 1775765415] req@ffff8f9970edc700 x1862022967120128/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765415.622484:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967120128/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765415.622489:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765415.622491:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765415.622522:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967120128:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765416.517961:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765417.541938:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765418.565894:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765419.589947:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765420.613906:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765420.741802:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765420.741812:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765420.741817:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765420.741828:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2042 00000100:00080000:3.0:1775765420.741839:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765420.741847:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765420.741881:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967120256/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765420.741910:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765420.741914:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765420.741915:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765420.741920:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2042 00000100:00080000:3.0:1775765420.741925:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765420.741929:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765420.741939:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765420.741954:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967120256:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765420.741955:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967120384/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765420.741986:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765420.741989:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765420.741992:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765420.741998:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2042 00000100:00080000:3.0:1775765420.742001:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765420.742005:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765420.742009:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765420.742022:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967120512/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765420.742031:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765420.742035:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765420.742037:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765420.742041:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2042 00000100:00080000:3.0:1775765420.742045:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765420.742049:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765420.742054:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765420.742057:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765420.742072:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967120640/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765420.742120:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765420/real 1775765420] req@ffff8f9970eded80 x1862022967120256/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765420.742144:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967120256/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765420.742157:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765420.742161:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765420.742164:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967120256:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765420.742188:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967120384:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765420.742205:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765420.742214:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765420/real 1775765420] req@ffff8f9970edc000 x1862022967120384/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765420.742221:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967120384/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765420.742226:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765420.742228:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765420.742230:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967120384:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765420.742312:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967120512:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765420.742328:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765420.742335:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765420/real 1775765420] req@ffff8f9970edce00 x1862022967120512/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765420.742342:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967120512/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765420.742348:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765420.742350:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765420.742351:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967120512:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765420.742355:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967120640:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765420.742365:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765420.742371:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765420/real 1775765420] req@ffff8f9970edd500 x1862022967120640/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765420.742377:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967120640/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765420.742382:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765420.742384:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765420.742386:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967120640:192.168.202.146@tcp:8:kworker.0 00010000:00010000:0.0:1775765421.637913:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765422.661965:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765423.685922:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765424.709909:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765425.733902:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765425.861805:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765425.861815:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765425.861821:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765425.861831:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2047 00000100:00080000:3.0:1775765425.861846:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765425.861854:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765425.861904:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf100 x1862022967120768/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765425.861933:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765425.861936:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765425.861938:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765425.861942:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2047 00000100:00080000:3.0:1775765425.861945:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765425.861950:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765425.861960:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765425.861967:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967120768:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765425.861972:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddc00 x1862022967120896/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765425.861981:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765425.861984:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765425.861985:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765425.861987:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2047 00000100:00080000:3.0:1775765425.861989:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765425.861991:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765425.861994:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765425.862002:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967121024/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765425.862008:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765425.862012:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765425.862013:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765425.862015:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2047 00000100:00080000:3.0:1775765425.862017:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765425.862020:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765425.862022:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765425.862028:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967121152/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765425.862153:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765425.862201:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765425/real 1775765425] req@ffff8f9970edf100 x1862022967120768/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765425.862216:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf100 x1862022967120768/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765425.862224:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765425.862227:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765425.862230:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967120768:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765425.862256:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967120896:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765425.862270:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765425.862278:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765425/real 1775765425] req@ffff8f9970eddc00 x1862022967120896/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765425.862285:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddc00 x1862022967120896/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765425.862289:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765425.862291:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765425.862293:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967120896:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765425.862297:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967121024:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765425.862308:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765425.862313:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765425/real 1775765425] req@ffff8f9970edd500 x1862022967121024/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765425.862353:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967121024/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765425.862358:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765425.862360:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765425.862362:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967121024:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765425.862365:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967121152:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765425.862378:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765425.862384:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765425/real 1775765425] req@ffff8f9970edce00 x1862022967121152/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765425.862391:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967121152/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765425.862395:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765425.862397:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765425.862399:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967121152:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765426.757883:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765427.781863:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765428.806043:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765429.830063:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765430.853958:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765430.981793:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765430.981802:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765430.981806:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765430.981816:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2052 00000100:00080000:3.0:1775765430.981826:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765430.981832:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765430.981864:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967121280/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765430.981889:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765430.981894:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765430.981896:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765430.981900:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2052 00000100:00080000:3.0:1775765430.981904:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765430.981908:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765430.981918:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765430.981922:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967121280:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765430.981932:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967121408/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765430.981942:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765430.981945:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765430.981946:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765430.981950:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2052 00000100:00080000:3.0:1775765430.981953:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765430.981955:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765430.981958:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765430.981968:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967121536/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765430.981975:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765430.981978:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765430.981980:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765430.981982:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2052 00000100:00080000:3.0:1775765430.981984:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765430.981987:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765430.981990:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765430.981993:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765430.982001:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967121664/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765430.982050:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765430/real 1775765430] req@ffff8f9970edc000 x1862022967121280/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765430.982076:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967121280/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765430.982147:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765430.982151:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765430.982154:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967121280:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765430.982175:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967121408:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765430.982196:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765430.982207:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765430/real 1775765430] req@ffff8f9970eded80 x1862022967121408/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765430.982216:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967121408/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765430.982222:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765430.982225:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765430.982227:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967121408:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765430.982231:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967121536:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765430.982243:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765430.982250:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765430/real 1775765430] req@ffff8f9970edc700 x1862022967121536/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765430.982258:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967121536/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765430.982264:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765430.982266:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765430.982268:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967121536:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765430.982287:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967121664:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765430.982300:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765430.982308:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765430/real 1775765430] req@ffff8f9970ede680 x1862022967121664/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765430.982316:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967121664/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765430.982322:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765430.982324:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765430.982327:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967121664:192.168.202.146@tcp:8:kworker.0 00010000:00010000:2.0:1775765431.877870:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765432.902042:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765433.926014:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765434.949921:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765435.973896:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765436.101903:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765436.101912:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765436.101917:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765436.101936:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2057 00000100:00080000:1.0:1775765436.101946:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765436.101952:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765436.102032:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967121792/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765436.102057:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765436.102061:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765436.102064:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765436.102070:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2057 00000100:00080000:1.0:1775765436.102075:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765436.102080:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765436.102090:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967121792:192.168.202.146@tcp:250:kworker.0 00000008:00100000:1.0:1775765436.102091:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765436.102111:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967121920/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765436.102125:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765436.102140:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765436.102142:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765436.102145:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2057 00000100:00080000:1.0:1775765436.102149:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775765436.102166:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775765436.102168:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765436.102174:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765436.102191:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967122048/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765436.102199:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765436.102203:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1775765436.102203:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765436/real 1775765436] req@ffff8f994f2e4700 x1862022967121792/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765436.102204:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765436.102214:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2057 00000100:00080000:1.0:1775765436.102222:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765436.102228:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765436.102230:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967121792/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1775765436.102236:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1775765436.102243:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765436.102249:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765436.102253:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967122176/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765436.102255:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967121792:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765436.102283:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967121920:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765436.102321:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765436.102330:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765436/real 1775765436] req@ffff8f994f2e4e00 x1862022967121920/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765436.102339:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967121920/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765436.102345:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765436.102347:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765436.102349:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967121920:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765436.102354:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967122048:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765436.102366:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765436.102372:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765436/real 1775765436] req@ffff8f994f2e6300 x1862022967122048/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765436.102379:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967122048/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765436.102384:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765436.102385:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765436.102387:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967122048:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765436.102391:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967122176:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765436.102403:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765436.102418:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765436/real 1775765436] req@ffff8f994f2e7b80 x1862022967122176/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765436.102465:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967122176/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765436.102485:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765436.102489:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765436.102493:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967122176:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775765436.998000:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765438.022009:0:5572:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765439.045953:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765440.069967:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765441.093936:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765441.221812:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765441.221820:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765441.221824:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765441.221832:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2062 00000100:00080000:3.0:1775765441.221842:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765441.221849:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765441.221881:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967122304/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765441.221903:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765441.221906:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765441.221907:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765441.221910:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2062 00000100:00080000:3.0:1775765441.221913:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765441.221915:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765441.221923:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765441.221934:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967122432/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765441.221936:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967122304:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775765441.221949:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765441.221951:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765441.221952:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765441.221954:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2062 00000100:00080000:3.0:1775765441.221955:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765441.221958:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765441.221961:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765441.221969:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967122560/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765441.221975:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765441.221978:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765441.221979:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765441.221981:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2062 00000100:00080000:3.0:1775765441.221986:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765441.221988:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765441.221991:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765441.221999:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765441.222000:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967122688/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765441.222045:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765441/real 1775765441] req@ffff8f9970edf800 x1862022967122304/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765441.222061:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967122304/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765441.222070:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765441.222073:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765441.222077:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967122304:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765441.222096:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967122432:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765441.222112:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765441.222120:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765441/real 1775765441] req@ffff8f9970ede300 x1862022967122432/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765441.222170:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967122432/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765441.222182:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765441.222184:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765441.222186:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967122432:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765441.222191:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967122560:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765441.222204:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765441.222211:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765441/real 1775765441] req@ffff8f9970edf480 x1862022967122560/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765441.222218:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967122560/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765441.222223:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765441.222225:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765441.222227:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967122560:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765441.222231:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967122688:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765441.222240:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765441.222245:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765441/real 1775765441] req@ffff8f9970edd880 x1862022967122688/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765441.222252:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967122688/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765441.222276:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765441.222279:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765441.222281:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967122688:192.168.202.146@tcp:8:kworker.0 00010000:00010000:0.0:1775765442.117979:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765443.141950:0:24627:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 11/11 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1775765443.142046:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765443.142046:0:5572:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 11 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775765443.142069:0:5572:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8f99523ee800 lock: ffff8f996695d600/0x36f9bdc9e421ca9b lrc: 2/0,0 mode: CR/CR res: [0x200000401:0xee2:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x2aac6a50e5fc736 expref: -99 pid: 24024 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000002:00100000:2.0:1775765443.142201:0:5572:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8f994f2e5880 x1862022967065600/t8589934642(8589934642) o101->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 1072/608 e 0 to 0 dl 1775764896 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775765443.142234:0:5572:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8f99523ee800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775765443.142239:0:5572:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8f99703d0a80 x1862022967122944/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/1048 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00080000:2.0:1775765443.142257:0:5572:0:(client.c:1830:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != DISCONN) req@ffff8f99703d0a80 x1862022967122944/t0(0) o35->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:23/10 lens 392/1048 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/600/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1775765443.142265:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775765444.165864:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765444.166067:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765445.189873:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765445.190019:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765446.213803:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765446.213875:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765446.341829:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765446.341840:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765446.341846:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765446.341857:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2067 00000100:00080000:3.0:1775765446.341871:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765446.341878:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765446.341923:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967123072/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765446.341949:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765446.341952:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765446.341954:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765446.341957:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2067 00000100:00080000:3.0:1775765446.341960:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765446.341963:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765446.341974:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765446.341987:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967123072:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765446.341987:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967123200/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765446.342001:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765446.342004:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765446.342004:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765446.342007:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2067 00000100:00080000:3.0:1775765446.342009:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765446.342011:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765446.342015:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765446.342025:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967123328/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765446.342031:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765446.342034:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765446.342036:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765446.342038:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2067 00000100:00080000:3.0:1775765446.342040:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765446.342043:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765446.342046:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765446.342053:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967123456/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765446.342146:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967123200:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765446.342469:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967123328:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765446.342507:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967123456:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765447.237821:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765447.237864:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765448.261796:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765448.261883:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765449.285965:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765449.286019:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000400:00000100:0.0:1775765449.414124:0:2367:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:0.0:1775765449.414195:0:2367:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1775765449.414219:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775765449.414291:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775765449.414337:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775765449.414366:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:2.0:1775765449.414444:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765446/real 1775765449] req@ffff8f9970edea00 x1862022967123200/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765449.414493:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967123200/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765449.414528:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765449.414538:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765449.414546:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967123200:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765449.414562:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765446/real 1775765449] req@ffff8f9970edca80 x1862022967123328/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765449.414597:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967123328/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765449.414613:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765449.414620:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765449.414627:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967123328:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765449.414640:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765446/real 1775765449] req@ffff8f9970edc380 x1862022967123456/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765449.414662:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967123456/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765449.414677:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765449.414682:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765449.414688:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967123456:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765449.414699:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765446/real 1775765449] req@ffff8f9970edfb80 x1862022967123072/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765449.414721:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967123072/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765449.414736:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765449.414785:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765449.414794:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967123072:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765450.309856:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765450.309971:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765451.333842:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765451.333923:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765451.461817:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765451.461824:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765451.461828:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765451.461838:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2073 00000100:00080000:3.0:1775765451.461847:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765451.461853:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765451.461883:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd180 x1862022967123584/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765451.461907:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765451.461911:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765451.461913:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765451.461917:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2073 00000100:00080000:3.0:1775765451.461921:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765451.461924:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765451.461935:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765451.461950:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddf80 x1862022967123712/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765451.461961:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967123584:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775765451.461965:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765451.461972:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765451.461973:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765451.461977:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2073 00000100:00080000:3.0:1775765451.461980:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765451.461984:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765451.461991:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765451.462006:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967123840/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765451.462021:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765451.462031:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765451.462033:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765451.462036:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2073 00000100:00080000:3.0:1775765451.462041:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775765451.462043:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775765451.462048:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765451.462054:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765451.462067:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967123968/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765451.462111:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765451/real 1775765451] req@ffff8f9970edd180 x1862022967123584/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765451.462134:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd180 x1862022967123584/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765451.462145:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765451.462150:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765451.462154:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967123584:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765451.462179:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967123712:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765451.462206:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765451.462220:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765451/real 1775765451] req@ffff8f9970eddf80 x1862022967123712/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765451.462243:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddf80 x1862022967123712/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765451.462252:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765451.462256:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765451.462260:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967123712:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765451.462267:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967123840:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765451.462319:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765451.462333:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765451/real 1775765451] req@ffff8f9970edfb80 x1862022967123840/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765451.462347:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967123840/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765451.462356:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765451.462360:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765451.462363:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967123840:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765451.462370:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967123968:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765451.462393:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765451.462405:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765451/real 1775765451] req@ffff8f9970edc380 x1862022967123968/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765451.462424:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967123968/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765451.462441:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765451.462445:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765451.462448:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967123968:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765452.357856:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765452.357990:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765453.381871:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765453.381993:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765454.405861:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765454.405981:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765455.429818:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765455.429875:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765456.453876:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765456.453922:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765456.581826:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765456.581834:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765456.581838:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765456.581847:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2078 00000100:00080000:3.0:1775765456.581858:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765456.581865:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765456.581901:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967124096/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765456.581930:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765456.581933:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765456.581935:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765456.581940:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2078 00000100:00080000:3.0:1775765456.581944:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765456.581948:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765456.581958:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765456.581960:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967124096:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765456.581974:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967124224/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765456.581984:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765456.581988:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765456.581990:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765456.581994:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2078 00000100:00080000:3.0:1775765456.582009:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765456.582013:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:2.0:1775765456.582019:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:3.0:1775765456.582021:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765456.582036:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967124352/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765456.582044:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765456.582047:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765456.582048:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765456.582051:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2078 00000100:00080000:3.0:1775765456.582054:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765456.582056:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765456.582060:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765456.582063:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765456/real 1775765456] req@ffff8f9970edca80 x1862022967124096/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765456.582097:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967124480/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765456.582107:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967124096/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765456.582136:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765456.582143:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765456.582150:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967124096:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765456.582196:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967124224:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765456.582221:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765456.582230:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765456/real 1775765456] req@ffff8f9970edea00 x1862022967124224/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765456.582258:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967124224/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765456.582264:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765456.582266:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765456.582268:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967124224:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765456.582273:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967124352:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765456.582288:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765456.582294:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765456/real 1775765456] req@ffff8f9970edd880 x1862022967124352/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765456.582302:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967124352/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765456.582307:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765456.582309:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765456.582311:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967124352:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765456.582315:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967124480:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765456.582324:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765456.582329:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765456/real 1775765456] req@ffff8f9970edf480 x1862022967124480/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765456.582336:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967124480/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765456.582382:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765456.582386:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765456.582390:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967124480:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765457.477814:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765457.477901:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765458.501841:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765458.501926:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765459.525812:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765459.525945:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765460.549876:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765460.549981:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765461.573872:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765461.573900:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765461.701893:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765461.701906:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765461.701913:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765461.701926:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2083 00000100:00080000:1.0:1775765461.701940:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765461.702143:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765461.702181:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967124608/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765461.702225:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765461.702229:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765461.702231:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765461.702235:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2083 00000100:00080000:1.0:1775765461.702245:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765461.702265:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765461.702274:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765461.702287:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967124608:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775765461.702289:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967124736/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765461.702311:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765461.702313:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765461.702315:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765461.702318:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2083 00000100:00080000:1.0:1775765461.702323:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765461.702327:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765461.702331:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765461.702338:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967124864/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765461.702344:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765461.702346:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765461.702364:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765461.702367:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2083 00000100:00080000:1.0:1775765461.702370:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765461.702374:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765461.702378:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765461.702385:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967124992/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765461.702395:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765461.702449:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765461/real 1775765461] req@ffff8f994f2e6680 x1862022967124608/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765461.702469:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967124608/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765461.702478:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765461.702481:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765461.702485:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967124608:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765461.702511:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967124736:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765461.702527:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765461.702536:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765461/real 1775765461] req@ffff8f994f2e5f80 x1862022967124736/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765461.702544:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967124736/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765461.702646:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765461.702651:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765461.702655:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967124736:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765461.702664:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967124864:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765461.702719:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765461.702741:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765461/real 1775765461] req@ffff8f994f2e6a00 x1862022967124864/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765461.702786:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967124864/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765461.702792:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765461.702795:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765461.702797:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967124864:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765461.702801:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967124992:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765461.702815:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765461.702827:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765461/real 1775765461] req@ffff8f994f2e4000 x1862022967124992/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765461.702835:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967124992/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765461.702840:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765461.702842:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765461.702856:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967124992:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765462.597872:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765462.598040:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765463.621798:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765463.622043:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765464.645834:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765464.645921:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765465.669801:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765465.669906:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765466.693810:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765466.693911:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765466.821826:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765466.821834:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765466.821838:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765466.821847:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2088 00000100:00080000:3.0:1775765466.821856:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765466.821862:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765466.821892:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967125120/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765466.821914:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765466.821917:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765466.821919:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765466.821922:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2088 00000100:00080000:3.0:1775765466.821925:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765466.821929:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765466.821939:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765466.821954:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967125120:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765466.821955:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967125248/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765466.821972:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765466.821976:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765466.821978:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765466.821983:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2088 00000100:00080000:3.0:1775765466.821987:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765466.821991:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765466.821995:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765466.822005:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967125376/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765466.822017:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765466.822021:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:2.0:1775765466.822022:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775765466.822023:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765466.822029:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2088 00000100:00080000:3.0:1775765466.822033:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765466.822035:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765466.822038:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765466.822046:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967125504/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765466.822081:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765466/real 1775765466] req@ffff8f9970ede300 x1862022967125120/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765466.822103:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967125120/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765466.822114:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765466.822119:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765466.822123:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967125120:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765466.822160:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967125248:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765466.822181:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765466.822190:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765466/real 1775765466] req@ffff8f9970edf800 x1862022967125248/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765466.822199:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967125248/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765466.822205:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765466.822207:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765466.822209:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967125248:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765466.822248:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967125376:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765466.822261:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765466.822268:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765466/real 1775765466] req@ffff8f9970ede680 x1862022967125376/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765466.822276:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967125376/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765466.822281:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765466.822283:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765466.822285:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967125376:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765466.822289:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967125504:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765466.822299:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765466.822304:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765466/real 1775765466] req@ffff8f9970edc700 x1862022967125504/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765466.822311:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967125504/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765466.822316:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765466.822318:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765466.822319:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967125504:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765467.717825:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765467.717969:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765468.741854:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765468.741998:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765469.765892:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765469.766162:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765470.789899:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765470.789998:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765471.813838:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765471.813923:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765471.942295:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765471.942307:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765471.942312:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765471.942326:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2093 00000100:00080000:3.0:1775765471.942340:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765471.942351:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765471.942399:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967125632/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765471.942620:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765471.942626:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765471.942630:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765471.942643:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2093 00000100:00080000:3.0:1775765471.942651:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765471.942657:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765471.942673:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765471.942697:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967125760/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765471.942700:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967125632:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775765471.942714:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765471.942719:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765471.942721:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1775765471.942864:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765471.942911:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765471/real 1775765471] req@ffff8f9970eded80 x1862022967125632/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765471.942928:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2093 00000100:00080000:3.0:1775765471.942936:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775765471.942939:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967125632/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765471.942943:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765471.942952:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1775765471.942956:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765471.942963:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765471.942971:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967125632:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765471.942972:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967125888/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765471.942987:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765471.942992:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765471.942995:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765471.943001:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2093 00000100:00100000:2.0:1775765471.943005:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967125760:192.168.202.146@tcp:38:kworker.0 00000100:00080000:3.0:1775765471.943007:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775765471.943050:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765471.943073:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765471/real 1775765471] req@ffff8f9970edc000 x1862022967125760/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765471.943088:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765471.943115:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967125760/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1775765471.943116:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1775765471.943129:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765471.943134:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765471.943135:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967126016/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765471.943139:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967125760:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765471.943149:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967125888:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765471.943182:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765471.943199:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765471/real 1775765471] req@ffff8f9970edce00 x1862022967125888/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765471.943241:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967125888/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765471.943250:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765471.943253:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765471.943257:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967125888:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765471.943285:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967126016:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765471.943317:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765471.943334:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765471/real 1775765471] req@ffff8f9970edd500 x1862022967126016/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765471.943351:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967126016/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765471.943364:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765471.943368:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765471.943373:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967126016:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765472.837862:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765472.837989:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765473.861888:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765473.861985:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765474.885873:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765474.885964:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765475.909887:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765475.910037:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765476.933873:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765476.934001:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765477.061866:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765477.061877:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765477.061884:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765477.061895:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2098 00000100:00080000:3.0:1775765477.061910:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765477.061918:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765477.061976:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddc00 x1862022967126144/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765477.062006:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765477.062010:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765477.062013:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765477.062017:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2098 00000100:00080000:3.0:1775765477.062021:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765477.062026:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765477.062037:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765477.062069:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967126144:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765477.062080:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf100 x1862022967126272/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765477.062093:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765477.062097:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765477.062099:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765477.062103:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2098 00000100:00080000:3.0:1775765477.062107:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765477.062112:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765477.062118:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765477.062141:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967126400/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765477.062152:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775765477.062170:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765477.062174:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765477.062176:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765477.062182:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2098 00000100:00080000:3.0:1775765477.062186:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765477.062191:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765477.062196:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765477.062213:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967126528/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765477.062224:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765477/real 1775765477] req@ffff8f9970eddc00 x1862022967126144/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765477.062304:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddc00 x1862022967126144/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765477.062315:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765477.062319:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765477.062323:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967126144:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765477.062348:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967126272:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765477.062372:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765477.062383:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765477/real 1775765477] req@ffff8f9970edf100 x1862022967126272/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765477.062396:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf100 x1862022967126272/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765477.062404:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765477.062407:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765477.062410:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967126272:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765477.062416:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967126400:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765477.062431:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765477.062439:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765477/real 1775765477] req@ffff8f9970edd500 x1862022967126400/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765477.062450:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967126400/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765477.062481:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765477.062485:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765477.062488:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967126400:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765477.062494:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967126528:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765477.062511:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765477.062521:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765477/real 1775765477] req@ffff8f9970edce00 x1862022967126528/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765477.062532:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967126528/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765477.062540:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765477.062543:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765477.062546:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967126528:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765477.957806:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765477.957895:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765478.981797:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765478.981888:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765480.005840:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765480.005885:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765481.029907:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765481.030056:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765482.053916:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765482.054023:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765482.181968:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765482.181980:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765482.181986:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765482.182021:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2103 00000100:00080000:1.0:1775765482.182036:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765482.182045:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765482.182091:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967126656/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765482.182130:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765482.182135:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765482.182175:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765482.182181:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2103 00000100:00080000:1.0:1775765482.182186:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765482.182192:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765482.182203:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967126656:192.168.202.146@tcp:250:kworker.0 00000008:00100000:1.0:1775765482.182204:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765482.182230:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967126784/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765482.182245:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765482.182249:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765482.182252:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765482.182258:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2103 00000100:00080000:1.0:1775765482.182262:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765482.182267:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765482.182273:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765482.182288:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967126912/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765482.182302:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775765482.182309:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765482.182314:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765482.182318:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765482.182325:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2103 00000100:00080000:1.0:1775765482.182333:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765482.182337:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765482.182344:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765482.182360:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765482/real 1775765482] req@ffff8f994f2e5180 x1862022967126656/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765482.182383:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967127040/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765482.182396:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967126656/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765482.182415:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765482.182455:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765482.182462:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967126656:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765482.182493:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967126784:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765482.182523:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765482.182539:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765482/real 1775765482] req@ffff8f994f2e7800 x1862022967126784/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765482.182554:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967126784/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765482.182565:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765482.182569:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765482.182573:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967126784:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765482.182580:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967126912:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765482.182599:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765482.182610:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765482/real 1775765482] req@ffff8f994f2e4a80 x1862022967126912/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765482.182624:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967126912/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765482.182633:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765482.182637:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765482.182640:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967126912:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765482.182647:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967127040:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765482.182693:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765482.182706:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765482/real 1775765482] req@ffff8f994f2e6d80 x1862022967127040/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765482.182721:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967127040/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765482.182730:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765482.182734:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765482.182768:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967127040:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765483.077837:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765483.077891:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765484.101886:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765484.102101:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765485.125871:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765485.125927:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765486.149873:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765486.149932:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765487.173834:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765487.174007:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765487.301932:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765487.301945:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765487.302006:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765487.302020:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2108 00000100:00080000:1.0:1775765487.302036:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765487.302056:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765487.302120:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967127168/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765487.302161:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765487.302166:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765487.302169:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765487.302174:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2108 00000100:00080000:1.0:1775765487.302178:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765487.302182:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765487.302195:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765487.302216:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967127296/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765487.302232:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765487.302243:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1775765487.302245:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967127168:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775765487.302280:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765487.302292:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2108 00000100:00080000:1.0:1775765487.302297:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765487.302302:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765487.302311:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765487.302328:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967127424/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765487.302343:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765487.302347:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765487.302350:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765487.302354:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2108 00000100:00080000:1.0:1775765487.302363:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775765487.302372:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775765487.302377:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765487.302387:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765487.302405:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967127552/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765487.302447:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765487/real 1775765487] req@ffff8f994f2e6d80 x1862022967127168/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765487.302475:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967127168/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765487.302489:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765487.302494:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765487.302498:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967127168:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765487.302531:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967127296:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765487.302555:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765487.302567:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765487/real 1775765487] req@ffff8f994f2e4a80 x1862022967127296/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765487.302880:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967127296/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765487.302896:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765487.302902:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765487.302908:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967127296:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765487.302922:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967127424:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765487.303039:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765487.303068:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765487/real 1775765487] req@ffff8f994f2e7800 x1862022967127424/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765487.303084:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967127424/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765487.303095:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765487.303099:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765487.303103:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967127424:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765487.303111:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967127552:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765487.303133:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765487.303145:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765487/real 1775765487] req@ffff8f994f2e5180 x1862022967127552/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765487.303159:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967127552/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765487.303203:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765487.303207:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765487.303211:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967127552:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765488.197818:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765488.198010:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765489.221870:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765489.222032:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765490.245910:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765490.246064:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765491.269885:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765491.270080:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765492.293913:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765492.293948:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765492.421996:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765492.422008:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765492.422013:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765492.422024:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2114 00000100:00080000:0.0:1775765492.422037:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765492.422222:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765492.422259:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967127680/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765492.422286:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765492.422289:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765492.422291:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765492.422296:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2114 00000100:00080000:0.0:1775765492.422306:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765492.422309:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765492.422317:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967127680:192.168.202.146@tcp:250:kworker.0 00000008:00100000:0.0:1775765492.422318:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765492.422333:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967127808/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765492.422346:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765492.422349:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765492.422351:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765492.422356:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2114 00000100:00080000:0.0:1775765492.422360:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765492.422365:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765492.422371:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765492.422378:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765492.422385:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967127936/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765492.422397:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765492.422399:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765492.422417:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765492.422421:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765492/real 1775765492] req@ffff8f9970a77100 x1862022967127680/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765492.422422:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2114 00000100:00080000:0.0:1775765492.422429:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765492.422435:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765492.422441:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765492.422445:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967127680/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765492.422455:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1775765492.422456:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967128064/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765492.422464:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765492.422471:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967127680:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765492.422493:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967127808:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765492.422509:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765492.422516:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765492/real 1775765492] req@ffff8f9970a76300 x1862022967127808/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765492.422524:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967127808/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765492.422530:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765492.422549:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765492.422551:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967127808:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765492.422556:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967127936:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765492.422569:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765492.422575:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765492/real 1775765492] req@ffff8f9970a75180 x1862022967127936/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765492.422583:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967127936/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765492.422588:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765492.422590:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765492.422592:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967127936:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765492.422596:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967128064:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765492.422605:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765492.422610:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765492/real 1775765492] req@ffff8f9970a75500 x1862022967128064/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765492.422616:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967128064/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765492.422621:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765492.422623:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765492.422625:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967128064:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765493.317938:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765493.318068:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765494.341844:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765494.342015:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765495.365866:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765495.365969:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765496.389826:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765496.389921:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765497.413924:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765497.413998:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765497.542023:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765497.542037:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765497.542061:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765497.542079:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2119 00000100:00080000:3.0:1775765497.542099:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765497.542127:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765497.542200:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967128192/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765497.542271:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765497.542275:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765497.542277:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765497.542283:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2119 00000100:00080000:3.0:1775765497.542287:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765497.542292:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765497.542307:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765497.542326:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967128320/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765497.542339:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765497.542343:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765497.542344:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765497.542350:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2119 00000100:00080000:3.0:1775765497.542353:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765497.542361:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765497.542368:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967128192:192.168.202.146@tcp:250:kworker.0 00000008:00100000:3.0:1775765497.542368:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765497.542384:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967128448/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765497.542394:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765497.542398:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765497.542399:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765497.542402:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2119 00000100:00080000:3.0:1775765497.542404:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765497.542408:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765497.542411:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765497.542423:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967128576/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765497.542502:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765497.542574:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765497/real 1775765497] req@ffff8f9970edc000 x1862022967128192/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765497.542603:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967128192/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765497.542619:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765497.542625:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765497.542630:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967128192:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765497.542663:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967128320:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765497.542702:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765497.542717:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765497/real 1775765497] req@ffff8f9970eded80 x1862022967128320/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765497.542732:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967128320/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765497.542790:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765497.542795:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765497.542799:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967128320:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765497.542807:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967128448:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765497.542833:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765497.542846:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765497/real 1775765497] req@ffff8f9970edc700 x1862022967128448/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765497.542973:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967128448/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765497.542991:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765497.542999:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765497.543006:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967128448:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765497.543018:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967128576:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765497.543060:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765497.543075:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765497/real 1775765497] req@ffff8f9970ede680 x1862022967128576/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765497.543090:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967128576/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765497.543101:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765497.543105:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765497.543109:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967128576:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765498.437891:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765498.438047:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765499.461919:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765499.462033:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765500.485952:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765500.485972:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765501.509954:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765501.510049:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765502.533876:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765502.533977:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765502.661902:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765502.662113:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765502.662123:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765502.662137:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2124 00000100:00080000:1.0:1775765502.662153:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765502.662161:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765502.662208:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967128704/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765502.662249:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765502.662264:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765502.662267:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765502.662273:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2124 00000100:00080000:1.0:1775765502.662279:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765502.662288:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765502.662303:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765502.662316:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967128704:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775765502.662331:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967128832/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765502.662354:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765502.662361:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765502.662365:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765502.662371:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2124 00000100:00080000:1.0:1775765502.662381:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765502.662389:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765502.662398:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765502.662401:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765502.662422:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967128960/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765502.662465:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765502.662469:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765502.662477:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765502.662477:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765502/real 1775765502] req@ffff8f994f2e4000 x1862022967128704/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765502.662491:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2124 00000100:00080000:1.0:1775765502.662502:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765502.662508:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765502.662514:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765502.662545:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967129088/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765502.662614:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967128704/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765502.662631:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765502.662637:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765502.662643:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967128704:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765502.662673:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967128832:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765502.662706:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765502.662722:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765502/real 1775765502] req@ffff8f994f2e6a00 x1862022967128832/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765502.662793:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967128832/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765502.662825:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765502.662830:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765502.662834:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967128832:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765502.662842:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967128960:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765502.662870:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765502.662883:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765502/real 1775765502] req@ffff8f994f2e5f80 x1862022967128960/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765502.662896:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967128960/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765502.662935:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765502.662939:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765502.662943:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967128960:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765502.662950:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967129088:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765502.662971:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765502.662983:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765502/real 1775765502] req@ffff8f994f2e6680 x1862022967129088/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765502.662996:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967129088/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765502.663005:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765502.663008:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765502.663011:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967129088:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765503.557892:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765503.557971:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765504.581922:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765504.582033:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765505.605896:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765505.605980:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765506.629881:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765506.629948:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765507.653862:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765507.653963:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765507.781881:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765507.781887:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765507.781892:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765507.781900:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2129 00000100:00080000:1.0:1775765507.781910:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765507.781916:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765507.781947:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967129216/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765507.781970:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765507.781973:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765507.781975:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765507.781978:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2129 00000100:00080000:1.0:1775765507.781997:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765507.782001:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765507.782006:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967129216:192.168.202.146@tcp:250:kworker.0 00000008:00100000:1.0:1775765507.782012:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765507.782026:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967129344/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765507.782035:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765507.782038:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765507.782040:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765507.782043:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2129 00000100:00080000:1.0:1775765507.782046:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765507.782049:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765507.782053:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765507.782060:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967129472/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765507.782066:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765507.782069:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765507.782071:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765507.782079:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2129 00000100:00080000:1.0:1775765507.782082:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765507.782085:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765507.782088:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765507.782095:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967129600/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775765507.782149:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765507.782283:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765507/real 1775765507] req@ffff8f994f2e7b80 x1862022967129216/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765507.782351:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967129216/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765507.782360:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765507.782363:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765507.782366:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967129216:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765507.782384:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967129344:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775765507.782403:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765507.782411:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765507/real 1775765507] req@ffff8f994f2e6300 x1862022967129344/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765507.782428:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967129344/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765507.782434:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765507.782436:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765507.782438:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967129344:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775765507.782442:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967129472:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775765507.782455:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765507.782461:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765507/real 1775765507] req@ffff8f994f2e4e00 x1862022967129472/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765507.782469:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967129472/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765507.782479:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765507.782481:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765507.782483:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967129472:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775765507.782486:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967129600:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775765507.782496:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765507.782501:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765507/real 1775765507] req@ffff8f994f2e4700 x1862022967129600/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765507.782508:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967129600/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765507.782513:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765507.782515:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765507.782517:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967129600:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765508.677856:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765508.677938:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765509.701850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765509.702028:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765510.725870:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765510.725958:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765511.749912:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765511.750025:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765512.773928:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765512.774035:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765512.901955:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765512.901974:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765512.901983:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765512.902002:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2134 00000100:00080000:2.0:1775765512.902021:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765512.902308:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765512.902372:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2d80 x1862022967129728/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765512.902427:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765512.902435:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765512.902448:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765512.902458:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2134 00000100:00080000:2.0:1775765512.902474:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765512.902483:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765512.902501:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765512.902531:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967129728:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765512.902532:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0700 x1862022967129856/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765512.902558:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765512.902562:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765512.902564:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765512.902570:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2134 00000100:00080000:2.0:1775765512.902576:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765512.902582:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765512.902591:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765512.902607:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0e00 x1862022967129984/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765512.902621:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765512.902626:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765512.902666:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765512.902676:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2134 00000100:00080000:2.0:1775765512.902686:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765512.902692:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765512.902726:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967129856:192.168.202.146@tcp:38:kworker.0 00000008:00100000:2.0:1775765512.902729:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765512.902808:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1500 x1862022967130112/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765512.902891:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967129984:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775765512.902984:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967130112:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765513.797890:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765513.797903:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765514.821866:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765514.821967:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765515.845817:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765515.845911:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775765515.974122:0:2369:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:1.0:1775765515.974184:0:2369:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775765515.974212:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775765515.974269:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775765515.974284:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775765515.974299:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:3.0:1775765515.974342:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765512/real 1775765515] req@ffff8f99703d1500 x1862022967130112/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765515.974403:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1500 x1862022967130112/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765515.974423:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765515.974430:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765515.974452:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967130112:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775765515.974464:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765512/real 1775765515] req@ffff8f99703d0700 x1862022967129856/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765515.974481:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0700 x1862022967129856/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765515.974491:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765515.974496:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765515.974501:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967129856:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775765515.974510:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765512/real 1775765515] req@ffff8f99703d0e00 x1862022967129984/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765515.974525:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0e00 x1862022967129984/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765515.974535:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765515.974539:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765515.974543:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967129984:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775765515.974553:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765512/real 1775765515] req@ffff8f99703d2d80 x1862022967129728/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765515.974569:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2d80 x1862022967129728/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765515.974579:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765515.974584:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765515.974588:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967129728:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765516.869841:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765516.869896:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765517.893823:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765517.893896:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765518.021909:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765518.021918:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765518.021923:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765518.021932:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2139 00000100:00080000:0.0:1775765518.021952:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765518.021958:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765518.021992:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967130240/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765518.022025:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765518.022029:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765518.022031:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765518.022036:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2139 00000100:00080000:0.0:1775765518.022040:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765518.022045:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765518.022055:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765518.022071:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967130368/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765518.022079:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967130240:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775765518.022092:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765518.022097:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765518.022099:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765518.022106:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2139 00000100:00080000:0.0:1775765518.022110:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765518.022114:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765518.022132:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765518.022144:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967130496/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765518.022156:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765518.022159:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765518.022161:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765518.022165:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2139 00000800:00000100:3.0:1775765518.022171:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775765518.022173:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765518.022179:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765518.022186:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765518.022200:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967130624/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765518.022228:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765518/real 1775765518] req@ffff8f9970a77480 x1862022967130240/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765518.022263:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967130240/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765518.022277:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765518.022282:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765518.022287:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967130240:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765518.022316:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967130368:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775765518.022343:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765518.022364:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765518/real 1775765518] req@ffff8f9970a77800 x1862022967130368/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765518.022381:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967130368/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765518.022390:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765518.022394:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765518.022397:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967130368:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775765518.022404:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967130496:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775765518.022427:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765518.022439:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765518/real 1775765518] req@ffff8f9970a76a00 x1862022967130496/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765518.022453:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967130496/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765518.022462:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765518.022465:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765518.022468:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967130496:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775765518.022475:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967130624:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775765518.022504:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765518.022517:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765518/real 1775765518] req@ffff8f9970a76d80 x1862022967130624/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765518.022603:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967130624/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765518.022613:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765518.022618:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765518.022621:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967130624:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765518.917849:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765518.917917:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765519.941815:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765519.941868:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765520.965876:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765520.965960:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765521.989854:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765521.989938:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765523.013900:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765523.013943:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765523.141907:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765523.141926:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765523.141937:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765523.141966:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2144 00000100:00080000:0.0:1775765523.141992:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765523.142032:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765523.142109:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967130752/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765523.142161:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765523.142176:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765523.142179:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765523.142183:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2144 00000100:00080000:0.0:1775765523.142187:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765523.142191:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765523.142214:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765523.142222:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967130752:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765523.142254:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967130880/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775765523.142307:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775765523.142335:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765523.142341:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765523.142345:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775765523.142383:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765523/real 1775765523] req@ffff8f9970a74380 x1862022967130752/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765523.142397:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2144 00000100:00100000:3.0:1775765523.142405:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967130752/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765523.142433:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765523.142437:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765523.142441:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967130752:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775765523.142443:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765523.142485:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765523.142501:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967130880:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775765523.142527:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:0.0:1775765523.142559:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765523.142572:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765523/real 1775765523] req@ffff8f9970a75c00 x1862022967130880/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765523.142597:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967131008/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765523.142633:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967130880/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765523.142653:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765523.142657:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765523.142660:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967130880:192.168.202.146@tcp:38:kworker.0 00000100:00080000:0.0:1775765523.142684:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765523.142688:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765523.142690:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765523.142696:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2144 00000100:00100000:3.0:1775765523.142700:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967131008:192.168.202.146@tcp:8:kworker.0 00000100:00080000:0.0:1775765523.142728:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1775765523.142773:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775765523.142802:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765523.142806:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765523/real 1775765523] req@ffff8f9970a74700 x1862022967131008/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765523.142818:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967131008/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1775765523.142832:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1775765523.142877:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765523.142884:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765523.142888:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967131008:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765523.142936:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967131136/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765523.143018:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967131136:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775765523.143062:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765523.143076:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765523/real 1775765523] req@ffff8f9970a74700 x1862022967131136/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765523.143090:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967131136/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765523.143098:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765523.143102:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765523.143105:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967131136:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765524.037896:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765524.037927:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765525.061905:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765525.061938:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765526.085826:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765526.085896:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765527.109848:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765527.109918:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765528.133880:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765528.133945:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765528.261881:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765528.261889:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765528.261893:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765528.261913:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2149 00000100:00080000:3.0:1775765528.261923:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765528.261929:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765528.261961:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967131264/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765528.261986:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765528.261989:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765528.261991:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765528.261994:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2149 00000100:00080000:3.0:1775765528.261997:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765528.262001:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765528.262012:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765528.262028:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967131264:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765528.262028:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967131392/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765528.262046:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765528.262050:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765528.262053:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765528.262058:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2149 00000100:00080000:3.0:1775765528.262062:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765528.262065:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765528.262070:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765528.262080:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967131520/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765528.262087:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1775765528.262092:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775765528.262092:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765528.262095:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765528.262099:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2149 00000100:00080000:3.0:1775765528.262102:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765528.262106:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765528.262110:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765528.262119:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967131648/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765528.262140:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765528/real 1775765528] req@ffff8f9970edf800 x1862022967131264/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765528.262157:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967131264/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765528.262165:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765528.262174:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765528.262177:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967131264:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765528.262246:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967131392:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765528.262280:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765528.262291:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765528/real 1775765528] req@ffff8f9970ede300 x1862022967131392/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765528.262301:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967131392/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765528.262309:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765528.262312:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765528.262314:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967131392:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765528.262336:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967131520:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765528.262351:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765528.262358:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765528/real 1775765528] req@ffff8f9970edf480 x1862022967131520/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765528.262367:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967131520/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765528.262373:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765528.262375:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765528.262378:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967131520:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765528.262382:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967131648:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765528.262393:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765528.262400:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765528/real 1775765528] req@ffff8f9970edd880 x1862022967131648/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765528.262423:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967131648/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765528.262430:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765528.262432:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765528.262435:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967131648:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765529.157848:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765529.157919:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765530.181875:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765530.181936:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765531.205831:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765531.205865:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765532.229839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765532.229932:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765533.253859:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765533.253971:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765533.381868:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765533.381885:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765533.381892:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765533.381908:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2154 00000100:00080000:3.0:1775765533.381926:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765533.381939:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765533.381994:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967131776/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765533.382048:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765533.382056:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765533.382060:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765533.382069:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2154 00000100:00080000:3.0:1775765533.382076:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765533.382081:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765533.382094:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765533.382117:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967131904/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765533.382127:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967131776:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775765533.382144:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765533.382149:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765533.382151:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765533.382157:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2154 00000100:00080000:3.0:1775765533.382164:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765533.382169:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765533.382176:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765533.382196:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967132032/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765533.382209:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765533.382214:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765533.382216:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765533.382220:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2154 00000100:00080000:3.0:1775765533.382233:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775765533.382240:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775765533.382246:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765533.382253:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765533.382269:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967132160/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765533.382320:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765533/real 1775765533] req@ffff8f9970edea00 x1862022967131776/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765533.382346:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967131776/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765533.382360:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765533.382365:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765533.382371:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967131776:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765533.382403:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967131904:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765533.382433:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765533.382449:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765533/real 1775765533] req@ffff8f9970edca80 x1862022967131904/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765533.382555:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967131904/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765533.382566:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765533.382571:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765533.382575:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967131904:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765533.382584:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967132032:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765533.382610:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765533.382623:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765533/real 1775765533] req@ffff8f9970edc380 x1862022967132032/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765533.382638:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967132032/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765533.382647:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765533.382650:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765533.382654:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967132032:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765533.382661:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967132160:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765533.382679:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765533.382689:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765533/real 1775765533] req@ffff8f9970edfb80 x1862022967132160/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765533.382702:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967132160/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765533.382786:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765533.382791:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765533.382795:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967132160:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765534.277867:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765534.277988:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765535.301897:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765535.301964:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765536.325860:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765536.326025:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765537.349866:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765537.350007:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765538.373812:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765538.373890:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765538.502013:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765538.502023:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765538.502028:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765538.502039:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2160 00000100:00080000:1.0:1775765538.502052:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765538.502060:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765538.502113:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967132288/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765538.502141:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765538.502146:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765538.502148:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765538.502153:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2160 00000100:00080000:1.0:1775765538.502194:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765538.502200:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765538.502210:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765538.502228:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967132416/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765538.502234:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967132288:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775765538.502249:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765538.502252:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765538.502254:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765538.502259:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2160 00000100:00080000:1.0:1775765538.502262:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765538.502265:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765538.502270:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765538.502281:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967132544/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765538.502290:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765538.502294:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765538.502297:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765538.502301:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2160 00000100:00080000:1.0:1775765538.502305:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765538.502311:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765538.502317:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765538.502320:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765538.502341:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967132672/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765538.502370:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765538/real 1775765538] req@ffff8f994f2e4700 x1862022967132288/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765538.502392:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967132288/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765538.502404:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765538.502408:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765538.502412:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967132288:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765538.502437:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967132416:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765538.502484:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765538.502499:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765538/real 1775765538] req@ffff8f994f2e4e00 x1862022967132416/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765538.502511:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967132416/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765538.502541:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765538.502545:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765538.502548:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967132416:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765538.502555:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967132544:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765538.502576:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765538.502587:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765538/real 1775765538] req@ffff8f994f2e6300 x1862022967132544/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765538.502600:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967132544/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765538.502621:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765538.502624:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765538.502628:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967132544:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765538.502634:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967132672:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765538.502655:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765538.502667:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765538/real 1775765538] req@ffff8f994f2e7b80 x1862022967132672/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765538.502678:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967132672/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765538.502701:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765538.502705:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765538.502724:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967132672:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765539.397860:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765539.397968:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765540.421875:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765540.421977:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765541.445870:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765541.445983:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765542.469875:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765542.470021:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765543.493816:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765543.493896:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765543.621909:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765543.621916:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765543.621920:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765543.621929:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2165 00000100:00080000:3.0:1775765543.621940:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765543.621946:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765543.621977:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddf80 x1862022967132800/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765543.621998:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765543.622003:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765543.622005:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765543.622008:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2165 00000100:00080000:3.0:1775765543.622011:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765543.622014:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765543.622023:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765543.622034:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd180 x1862022967132928/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765543.622044:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967132800:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775765543.622044:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765543.622049:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765543.622053:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765543.622058:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2165 00000100:00080000:3.0:1775765543.622060:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765543.622062:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765543.622065:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765543.622074:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967133056/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765543.622080:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765543.622082:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765543.622084:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765543.622086:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2165 00000100:00080000:3.0:1775765543.622089:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765543.622091:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765543.622095:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765543.622103:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967133184/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765543.622109:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765543.622157:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765543/real 1775765543] req@ffff8f9970eddf80 x1862022967132800/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765543.622170:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddf80 x1862022967132800/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765543.622178:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765543.622181:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765543.622184:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967132800:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775765543.622202:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967132928:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775765543.622216:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765543.622224:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765543/real 1775765543] req@ffff8f9970edd180 x1862022967132928/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765543.622242:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd180 x1862022967132928/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765543.622248:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765543.622250:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765543.622252:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967132928:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775765543.622256:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967133056:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765543.622268:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765543.622313:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765543/real 1775765543] req@ffff8f9970edfb80 x1862022967133056/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765543.622327:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967133056/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765543.622335:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765543.622339:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765543.622342:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967133056:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775765543.622348:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967133184:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765543.622381:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765543.622391:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765543/real 1775765543] req@ffff8f9970edc380 x1862022967133184/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765543.622401:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967133184/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765543.622407:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765543.622426:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765543.622428:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967133184:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765544.517881:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765544.517971:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765545.541908:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765545.542017:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765546.565863:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765546.566026:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765547.589882:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765547.590031:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765548.613886:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765548.614015:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765548.741922:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765548.741934:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765548.741940:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765548.741952:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2170 00000100:00080000:1.0:1775765548.741966:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765548.741973:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765548.742084:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967133312/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765548.742121:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765548.742126:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765548.742129:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765548.742133:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2170 00000100:00080000:1.0:1775765548.742138:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765548.742142:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765548.742153:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765548.742166:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967133440/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765548.742178:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765548.742182:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765548.742185:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765548.742189:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2170 00000100:00080000:1.0:1775765548.742193:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765548.742197:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765548.742203:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765548.742213:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967133568/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765548.742225:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765548.742229:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765548.742232:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765548.742238:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967133312:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775765548.742260:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2170 00000100:00080000:1.0:1775765548.742266:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765548.742270:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765548.742276:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765548.742293:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967133696/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765548.742327:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765548.742435:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765548/real 1775765548] req@ffff8f994f2e6680 x1862022967133312/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765548.742459:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967133312/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765548.742474:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765548.742481:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765548.742487:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967133312:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765548.742498:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967133440:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765548.742535:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765548.742550:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765548/real 1775765548] req@ffff8f994f2e5f80 x1862022967133440/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765548.742564:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967133440/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765548.742602:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765548.742606:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765548.742610:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967133440:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765548.742641:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967133568:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765548.742664:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765548.742676:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765548/real 1775765548] req@ffff8f994f2e6a00 x1862022967133568/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765548.742689:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967133568/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765548.742698:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765548.742702:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765548.742705:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967133568:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765548.742711:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967133696:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765548.742728:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765548.742768:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765548/real 1775765548] req@ffff8f994f2e4000 x1862022967133696/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765548.742783:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967133696/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765548.742792:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765548.742795:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765548.742819:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967133696:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765549.637867:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765549.638060:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765550.661906:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765550.662038:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765551.685887:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765551.685974:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765552.709874:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765552.710035:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765553.733904:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765553.734000:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765553.861969:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765553.861985:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765553.861993:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765553.862009:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2175 00000100:00080000:3.0:1775765553.862024:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765553.862032:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765553.862080:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967133824/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765553.862130:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765553.862136:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765553.862139:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765553.862145:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2175 00000100:00080000:3.0:1775765553.862160:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765553.862165:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765553.862177:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765553.862193:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967133952/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765553.862210:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967133824:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775765553.862226:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765553.862232:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765553.862234:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765553.862242:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2175 00000100:00080000:3.0:1775765553.862256:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765553.862260:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765553.862268:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765553.862285:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967134080/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765553.862298:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765553.862303:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765553.862305:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765553.862309:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2175 00000100:00080000:3.0:1775765553.862338:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765553.862343:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765553.862350:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765553.862368:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765553.862385:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967134208/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765553.862440:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765553/real 1775765553] req@ffff8f9970edca80 x1862022967133824/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765578 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765553.862469:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967133824/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765578 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765553.862485:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765553.862491:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765553.862496:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967133824:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765553.862525:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967133952:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765553.862554:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765553.862567:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765553/real 1775765553] req@ffff8f9970edea00 x1862022967133952/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765578 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765553.862582:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967133952/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765578 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765553.862592:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765553.862596:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765553.862657:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967133952:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765553.862665:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967134080:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765553.862688:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765553.862701:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765553/real 1775765553] req@ffff8f9970edd880 x1862022967134080/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765578 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765553.862716:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967134080/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765578 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765553.862726:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765553.862730:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765553.862734:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967134080:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765553.862795:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967134208:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765553.862820:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765553.862833:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765553/real 1775765553] req@ffff8f9970edf480 x1862022967134208/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765578 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765553.862848:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967134208/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765578 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765553.862858:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765553.862862:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765553.862866:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967134208:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765554.757871:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765554.758019:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765555.781894:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765555.782036:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765556.805916:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765556.805984:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765557.829887:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765557.830015:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765558.853955:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765558.854068:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765558.981851:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765558.981862:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765558.981867:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765558.981878:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2180 00000100:00080000:3.0:1775765558.981890:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765558.981896:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765558.981946:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967134336/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765558.981976:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765558.981981:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765558.981982:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765558.981986:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2180 00000100:00080000:3.0:1775765558.981989:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765558.981992:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765558.982000:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765558.982012:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967134464/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765558.982021:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765558.982025:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1775765558.982026:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967134336:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775765558.982026:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765558.982033:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2180 00000100:00080000:3.0:1775765558.982036:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765558.982039:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765558.982042:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765558.982054:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967134592/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765558.982064:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765558.982068:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765558.982070:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765558.982074:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2180 00000100:00080000:3.0:1775765558.982078:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765558.982082:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765558.982089:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765558.982090:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765558.982105:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967134720/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765558.982138:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765558/real 1775765558] req@ffff8f9970ede300 x1862022967134336/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765558.982153:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967134336/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765558.982162:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765558.982166:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765558.982171:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967134336:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765558.982189:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967134464:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765558.982203:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765558.982211:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765558/real 1775765558] req@ffff8f9970edf800 x1862022967134464/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765558.982219:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967134464/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765558.982225:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765558.982227:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765558.982230:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967134464:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765558.982233:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967134592:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765558.982319:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765558.982327:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765558/real 1775765558] req@ffff8f9970ede680 x1862022967134592/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765558.982335:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967134592/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765558.982341:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765558.982343:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765558.982345:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967134592:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765558.982356:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967134720:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765558.982368:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765558.982373:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765558/real 1775765558] req@ffff8f9970edc700 x1862022967134720/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765558.982381:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967134720/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765558.982386:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765558.982388:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765558.982390:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967134720:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765559.877946:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765559.878059:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765560.901929:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765560.901948:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765561.925910:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765561.926070:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765562.949828:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765562.949954:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765563.973881:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765563.973984:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765564.101849:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765564.101863:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765564.101869:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765564.101894:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2185 00000100:00080000:3.0:1775765564.101909:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765564.101919:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765564.101978:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967134848/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765564.102021:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765564.102027:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765564.102030:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765564.102037:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2185 00000100:00080000:3.0:1775765564.102042:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765564.102046:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765564.102058:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765564.102073:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967134976/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765564.102086:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765564.102089:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765564.102091:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765564.102095:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2185 00000100:00080000:3.0:1775765564.102100:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765564.102105:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765564.102111:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765564.102126:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967135104/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765564.102138:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765564.102142:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765564.102145:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765564.102149:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2185 00000100:00080000:3.0:1775765564.102154:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775765564.102161:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967134848:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775765564.102164:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765564.102170:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765564.102181:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967135232/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765564.102337:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765564.102389:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765564/real 1775765564] req@ffff8f9970eded80 x1862022967134848/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765564.102409:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967134848/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765564.102420:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765564.102424:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765564.102428:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967134848:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765564.102437:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967134976:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765564.102460:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765564.102472:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765564/real 1775765564] req@ffff8f9970edc000 x1862022967134976/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765564.102483:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967134976/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765564.102490:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765564.102494:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765564.102497:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967134976:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765564.102502:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967135104:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765564.102516:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765564.102524:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765564/real 1775765564] req@ffff8f9970edce00 x1862022967135104/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765564.102561:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967135104/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765564.102569:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765564.102572:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765564.102575:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967135104:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765564.102630:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967135232:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765564.102656:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765564.102669:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765564/real 1775765564] req@ffff8f9970edd500 x1862022967135232/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765564.102683:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967135232/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765564.102693:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765564.102697:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765564.102700:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967135232:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765564.997820:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765564.997868:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765566.021810:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765566.021861:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765567.045884:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765567.045961:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765568.069849:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765568.069911:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765569.093914:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765569.093968:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765569.221854:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765569.221863:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765569.221867:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765569.221878:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2190 00000100:00080000:3.0:1775765569.221888:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765569.221894:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765569.221931:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf100 x1862022967135360/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765569.221971:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765569.221976:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765569.221978:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765569.221982:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2190 00000100:00080000:3.0:1775765569.221984:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765569.221987:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765569.221996:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765569.222006:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddc00 x1862022967135488/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765569.222016:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765569.222019:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765569.222021:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765569.222027:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967135360:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775765569.222027:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2190 00000100:00080000:3.0:1775765569.222033:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765569.222036:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765569.222040:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765569.222066:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967135616/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765569.222074:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765569.222078:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765569.222080:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765569.222082:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2190 00000100:00080000:3.0:1775765569.222085:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765569.222088:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765569.222092:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765569.222101:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967135744/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765569.222148:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765569.222256:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765569/real 1775765569] req@ffff8f9970edf100 x1862022967135360/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765569.222312:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf100 x1862022967135360/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765569.222340:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765569.222344:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765569.222347:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967135360:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765569.222369:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967135488:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765569.222390:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765569.222401:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765569/real 1775765569] req@ffff8f9970eddc00 x1862022967135488/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765569.222411:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddc00 x1862022967135488/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765569.222418:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765569.222420:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765569.222423:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967135488:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765569.222427:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967135616:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765569.222440:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765569.222448:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765569/real 1775765569] req@ffff8f9970edd500 x1862022967135616/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765569.222457:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967135616/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765569.222485:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765569.222488:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765569.222491:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967135616:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765569.222496:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967135744:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765569.222510:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765569.222535:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765569/real 1775765569] req@ffff8f9970edce00 x1862022967135744/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765569.222544:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967135744/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765569.222550:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765569.222552:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765569.222555:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967135744:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765570.117825:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765570.117870:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765571.141933:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765571.142008:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765572.165887:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765572.165980:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765573.189878:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765573.189985:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765574.213896:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765574.213986:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765574.341814:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765574.341834:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765574.341837:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765574.341846:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2195 00000100:00080000:3.0:1775765574.341856:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765574.341861:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765574.341905:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967135872/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765574.341927:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765574.341929:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765574.341931:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765574.341934:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2195 00000100:00080000:3.0:1775765574.341937:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765574.341939:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765574.341947:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765574.341955:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967136000/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765574.341960:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967135872:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775765574.341967:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765574.341970:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765574.341971:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765574.341974:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2195 00000100:00080000:3.0:1775765574.341977:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765574.341980:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765574.341983:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765574.341993:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967136128/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765574.341999:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765574.342002:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765574.342003:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765574.342005:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2195 00000100:00080000:3.0:1775765574.342015:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765574.342017:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765574.342019:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765574.342039:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967136256/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765574.342040:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765574.342095:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765574/real 1775765574] req@ffff8f9970edc000 x1862022967135872/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765574.342112:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967135872/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765574.342175:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765574.342179:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765574.342182:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967135872:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765574.342200:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967136000:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765574.342219:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765574.342227:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765574/real 1775765574] req@ffff8f9970eded80 x1862022967136000/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765574.342236:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967136000/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765574.342241:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765574.342243:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765574.342245:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967136000:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765574.342249:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967136128:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765574.342259:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765574.342264:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765574/real 1775765574] req@ffff8f9970edc700 x1862022967136128/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765574.342271:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967136128/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765574.342276:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765574.342278:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765574.342280:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967136128:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765574.342303:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967136256:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765574.342315:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765574.342321:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765574/real 1775765574] req@ffff8f9970ede680 x1862022967136256/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765574.342328:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967136256/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765574.342333:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765574.342335:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765574.342337:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967136256:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765575.237833:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765575.237894:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765576.261803:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765576.261866:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765577.285827:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765577.285881:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765578.309825:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765578.309883:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765579.333858:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765579.333933:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765579.461796:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765579.461803:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765579.461808:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765579.461816:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2201 00000100:00080000:3.0:1775765579.461825:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765579.461831:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765579.461861:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967136384/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765579.461884:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765579.461889:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765579.461891:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765579.461894:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2201 00000100:00080000:3.0:1775765579.461897:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765579.461900:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765579.461908:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765579.461918:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967136384:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765579.461919:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967136512/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765579.461933:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765579.461936:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765579.461937:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765579.461939:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2201 00000100:00080000:3.0:1775765579.461942:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765579.461944:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765579.461947:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765579.461955:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967136640/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765579.461962:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765579.461965:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765579.461966:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765579.461968:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2201 00000100:00080000:3.0:1775765579.461972:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765579.461974:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765579.461978:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765579.461987:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967136768/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765579.462039:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967136512:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765579.462142:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967136640:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765579.462176:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967136768:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765580.357839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765580.358004:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765581.381840:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765581.381910:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765582.405833:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765582.405919:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1775765582.534119:0:2366:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:0.0:1775765582.534175:0:2366:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1775765582.534198:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775765582.534249:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775765582.534264:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775765582.534297:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:2.0:1775765582.534367:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765579/real 1775765582] req@ffff8f9970ede300 x1862022967136512/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765582.534474:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967136512/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765582.534492:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765582.534499:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765582.534505:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967136512:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765582.534516:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765579/real 1775765582] req@ffff8f9970edf480 x1862022967136640/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765582.534535:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967136640/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765582.534546:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765582.534551:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765582.534556:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967136640:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765582.534565:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765579/real 1775765582] req@ffff8f9970edd880 x1862022967136768/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765582.534583:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967136768/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765582.534596:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765582.534602:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765582.534607:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967136768:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765582.534617:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765579/real 1775765582] req@ffff8f9970edf800 x1862022967136384/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765582.534671:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967136384/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765582.534684:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765582.534689:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765582.534695:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967136384:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765583.429840:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765583.429912:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765584.453853:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765584.453921:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765584.581959:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765584.581974:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765584.581980:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765584.581995:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2206 00000100:00080000:0.0:1775765584.582036:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765584.582047:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765584.582102:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967136896/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765584.582130:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765584.582134:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765584.582136:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765584.582150:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2206 00000100:00080000:0.0:1775765584.582154:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765584.582157:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765584.582167:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765584.582180:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967137024/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765584.582191:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967136896:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775765584.582248:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765584.582253:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765584.582256:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765584.582265:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2206 00000100:00080000:0.0:1775765584.582285:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775765584.582292:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775765584.582295:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765584.582304:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765584.582324:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967137152/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765584.582337:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765584.582341:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765584.582344:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765584.582349:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2206 00000100:00080000:0.0:1775765584.582355:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765584.582360:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765584.582367:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765584/real 1775765584] req@ffff8f9970a75c00 x1862022967136896/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1775765584.582368:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765584.582390:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967137280/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765584.582402:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967136896/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765584.582420:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765584.582428:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765584.582435:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967136896:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765584.582484:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967137024:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765584.582518:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765584.582537:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765584/real 1775765584] req@ffff8f9970a74380 x1862022967137024/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765584.582556:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967137024/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765584.582597:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765584.582602:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765584.582606:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967137024:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765584.582616:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967137152:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765584.582647:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765584.582662:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765584/real 1775765584] req@ffff8f9970a76d80 x1862022967137152/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765584.582678:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967137152/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765584.582689:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765584.582694:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765584.582709:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967137152:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765584.582718:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967137280:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765584.582777:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765584.582794:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765584/real 1775765584] req@ffff8f9970a76a00 x1862022967137280/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765584.582810:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967137280/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765584.582822:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765584.582828:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765584.582855:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967137280:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765585.477810:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765585.477871:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765586.501859:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765586.501923:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765587.525836:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765587.525949:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765588.549866:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765588.549978:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765589.573839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765589.573936:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765589.701885:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765589.701898:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765589.701904:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765589.701943:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2211 00000100:00080000:3.0:1775765589.701957:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765589.701963:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765589.702031:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967137408/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765589.702066:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765589.702069:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765589.702071:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765589.702075:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2211 00000100:00080000:3.0:1775765589.702079:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765589.702081:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765589.702091:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765589.702102:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967137536/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765589.702112:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775765589.702115:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967137408:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775765589.702116:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765589.702129:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765589.702139:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2211 00000100:00080000:3.0:1775765589.702142:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765589.702145:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765589.702149:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765589.702162:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967137664/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765589.702169:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765589.702172:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765589.702174:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765589.702177:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2211 00000100:00080000:3.0:1775765589.702201:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765589.702204:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765589.702208:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765589.702238:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765589.702238:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967137792/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765589.702312:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765589/real 1775765589] req@ffff8f9970edea00 x1862022967137408/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765589.702353:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967137408/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765589.702367:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765589.702372:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765589.702378:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967137408:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765589.702404:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967137536:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765589.702432:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765589.702448:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765589/real 1775765589] req@ffff8f9970edca80 x1862022967137536/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765589.702464:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967137536/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765589.702473:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765589.702477:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765589.702527:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967137536:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765589.702535:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967137664:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765589.702562:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765589.702576:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765589/real 1775765589] req@ffff8f9970edc380 x1862022967137664/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765589.702591:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967137664/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765589.702602:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765589.702606:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765589.702610:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967137664:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765589.702617:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967137792:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765589.702635:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765589.702645:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765589/real 1775765589] req@ffff8f9970edfb80 x1862022967137792/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765589.702659:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967137792/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765589.702666:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765589.702668:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765589.702671:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967137792:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765590.597864:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765590.597994:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765591.621882:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765591.622025:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765592.645844:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765592.645935:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765593.669887:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765593.670056:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765594.693797:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765594.693956:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765594.821949:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765594.821991:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765594.821998:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765594.822012:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2216 00000100:00080000:1.0:1775765594.822030:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765594.822039:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765594.822097:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967137920/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765594.822158:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765594.822162:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765594.822164:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765594.822168:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2216 00000100:00080000:1.0:1775765594.822181:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765594.822184:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765594.822197:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765594.822212:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967138048/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765594.822221:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967137920:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775765594.822227:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1775765594.822314:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765594.822399:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765594/real 1775765594] req@ffff8f994f2e5180 x1862022967137920/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765594.822433:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967137920/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765594.822459:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765594.822461:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765594.822464:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765594.822471:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00080000:1.0:1775765594.822482:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2216 00000100:00100000:2.0:1775765594.822485:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967137920:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775765594.822507:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765594.822518:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765594.822529:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765594.822554:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967138048:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775765594.822572:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967138176/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765594.822597:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1775765594.822601:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775765594.822608:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765594.822613:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765594.822621:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2216 00000100:00100000:2.0:1775765594.822626:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765594/real 1775765594] req@ffff8f994f2e7800 x1862022967138048/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765594.822628:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765594.822661:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765594.822667:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967138048/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1775765594.822672:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1775765594.822692:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765594.822699:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765594.822704:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967138304/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765594.822710:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967138048:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765594.822833:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967138176:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765594.823120:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765594.823175:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765594/real 1775765594] req@ffff8f994f2e4a80 x1862022967138176/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765594.823192:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967138176/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765594.823202:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765594.823206:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765594.823209:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967138176:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765594.823219:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967138304:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765594.823235:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765594.823243:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765594/real 1775765594] req@ffff8f994f2e6d80 x1862022967138304/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765594.823253:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967138304/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765594.823259:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765594.823262:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765594.823264:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967138304:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765595.717828:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765595.717897:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765596.741936:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765596.742044:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765597.765859:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765597.765955:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765598.789876:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765598.789973:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765599.813801:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765599.813867:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765599.941926:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765599.941936:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765599.941941:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765599.941952:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2221 00000100:00080000:3.0:1775765599.941964:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765599.941971:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765599.942021:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd180 x1862022967138432/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765599.942053:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765599.942060:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765599.942073:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765599.942078:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2221 00000100:00080000:3.0:1775765599.942082:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765599.942087:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765599.942098:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765599.942115:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddf80 x1862022967138560/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765599.942127:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967138432:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775765599.942131:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765599.942135:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765599.942137:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765599.942142:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2221 00000100:00080000:3.0:1775765599.942146:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765599.942151:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765599.942157:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765599.942172:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967138688/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765599.942184:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765599.942188:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765599.942191:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765599.942195:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2221 00000100:00080000:3.0:1775765599.942200:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765599.942204:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765599.942211:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765599.942225:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967138816/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765599.942236:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765599.942294:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765599/real 1775765599] req@ffff8f9970edd180 x1862022967138432/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765599.942311:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd180 x1862022967138432/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765599.942320:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765599.942325:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765599.942329:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967138432:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765599.942349:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967138560:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765599.942367:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765599.942376:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765599/real 1775765599] req@ffff8f9970eddf80 x1862022967138560/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765599.942385:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddf80 x1862022967138560/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765599.942391:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765599.942393:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765599.942395:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967138560:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765599.942399:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967138688:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765599.942411:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765599.942417:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765599/real 1775765599] req@ffff8f9970edfb80 x1862022967138688/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765599.942446:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967138688/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765599.942452:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765599.942455:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765599.942457:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967138688:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765599.942461:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967138816:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765599.942486:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765599.942495:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765599/real 1775765599] req@ffff8f9970edc380 x1862022967138816/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765599.942508:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967138816/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765599.942518:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765599.942521:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765599.942525:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967138816:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765600.837874:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765600.837938:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765601.861823:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765601.861967:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765602.885981:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765602.886057:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765603.909878:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765603.909927:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765604.933847:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765604.933923:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765605.062048:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765605.062067:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765605.062073:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765605.062293:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2226 00000100:00080000:2.0:1775765605.062306:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765605.062314:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765605.062353:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3800 x1862022967138944/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765605.062386:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765605.062390:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765605.062393:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765605.062398:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2226 00000100:00080000:2.0:1775765605.062403:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765605.062407:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765605.062417:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765605.062426:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967138944:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765605.062432:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2680 x1862022967139072/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765605.062451:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765605.062454:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765605.062455:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765605.062459:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2226 00000100:00080000:2.0:1775765605.062464:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765605.062467:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765605.062473:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765605.062494:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0000 x1862022967139200/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765605.062507:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775765605.062532:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765605.062536:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765605.062539:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775765605.062545:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765605/real 1775765605] req@ffff8f99703d3800 x1862022967138944/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765605.062546:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2226 00000100:00080000:2.0:1775765605.062553:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765605.062560:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765605.062566:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765605.062569:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3800 x1862022967138944/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765605.062583:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0380 x1862022967139328/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765605.062601:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765605.062606:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765605.062610:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967138944:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775765605.062628:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967139072:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775765605.062646:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765605.062654:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765605/real 1775765605] req@ffff8f99703d2680 x1862022967139072/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765605.062668:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2680 x1862022967139072/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765605.062673:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765605.062675:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765605.062678:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967139072:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775765605.062681:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967139200:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765605.062693:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765605.062699:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765605/real 1775765605] req@ffff8f99703d0000 x1862022967139200/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765605.062706:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0000 x1862022967139200/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765605.062711:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765605.062713:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765605.062715:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967139200:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775765605.062731:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967139328:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765605.062787:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765605.062794:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765605/real 1775765605] req@ffff8f99703d0380 x1862022967139328/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765605.062802:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0380 x1862022967139328/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765605.062808:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765605.062810:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765605.062812:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967139328:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765605.957870:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765605.957918:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765606.981827:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765606.981866:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765608.005904:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765608.005999:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765609.029944:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765609.030032:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765610.054005:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765610.054025:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765610.181897:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765610.181907:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765610.181912:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765610.181922:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2231 00000100:00080000:2.0:1775765610.181934:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765610.181941:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765610.181987:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0380 x1862022967139456/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765610.182020:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765610.182024:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765610.182068:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765610.182075:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2231 00000100:00080000:2.0:1775765610.182079:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765610.182084:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765610.182095:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765610.182108:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0000 x1862022967139584/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765610.182128:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967139456:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775765610.182128:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765610.182163:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765610.182169:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765610.182176:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2231 00000100:00080000:2.0:1775765610.182180:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765610.182184:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765610.182191:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765610.182209:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2680 x1862022967139712/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765610.182220:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765610.182224:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765610.182226:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765610.182229:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2231 00000100:00080000:2.0:1775765610.182234:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765610.182240:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765610.182245:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765610.182274:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3800 x1862022967139840/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765610.182294:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765610.182378:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765610/real 1775765610] req@ffff8f99703d0380 x1862022967139456/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765610.182409:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0380 x1862022967139456/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765610.182437:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765610.182444:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765610.182449:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967139456:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775765610.182485:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967139584:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775765610.182515:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765610.182531:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765610/real 1775765610] req@ffff8f99703d0000 x1862022967139584/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765610.182919:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0000 x1862022967139584/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765610.182932:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765610.182945:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765610.182950:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967139584:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775765610.182961:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967139712:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765610.183020:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765610.183044:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765610/real 1775765610] req@ffff8f99703d2680 x1862022967139712/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765610.183055:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2680 x1862022967139712/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765610.183062:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765610.183065:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765610.183068:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967139712:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775765610.183073:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967139840:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765610.183087:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765610.183095:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765610/real 1775765610] req@ffff8f99703d3800 x1862022967139840/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765610.183105:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3800 x1862022967139840/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765610.183144:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765610.183147:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765610.183150:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967139840:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765611.077898:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765611.077945:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765612.101901:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765612.102013:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765613.125945:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765613.126044:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765614.149946:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765614.150000:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765615.173844:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765615.173890:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765615.301996:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765615.302009:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765615.302014:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765615.302027:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2236 00000100:00080000:0.0:1775765615.302041:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765615.302051:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765615.302157:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967139968/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765615.302211:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765615.302217:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765615.302220:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765615.302227:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2236 00000100:00080000:0.0:1775765615.302235:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765615.302242:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765615.302254:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765615.302275:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967140096/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765615.302289:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967139968:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775765615.302298:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765615.302309:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765615.302312:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765615.302320:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2236 00000100:00080000:0.0:1775765615.302326:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765615.302332:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765615.302340:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765615.302360:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967140224/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765615.302373:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765615.302378:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765615.302381:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1775765615.302397:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775765615.302411:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2236 00000100:00080000:0.0:1775765615.302419:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765615.302425:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765615.302432:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765615.302452:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967140352/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765615.302463:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765615/real 1775765615] req@ffff8f9970a77800 x1862022967139968/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765615.302496:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967139968/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765615.302512:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765615.302518:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765615.302524:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967139968:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775765615.302554:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967140096:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775765615.302581:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765615.302595:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765615/real 1775765615] req@ffff8f9970a77480 x1862022967140096/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765615.302612:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967140096/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765615.302623:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765615.302629:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765615.302633:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967140096:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775765615.302679:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967140224:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765615.302707:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765615.302805:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765615/real 1775765615] req@ffff8f9970a75500 x1862022967140224/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765615.302843:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967140224/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765615.302863:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765615.302871:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765615.302876:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967140224:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775765615.302889:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967140352:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765615.302947:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765615.302991:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765615/real 1775765615] req@ffff8f9970a75180 x1862022967140352/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765615.303021:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967140352/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765615.303039:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765615.303047:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765615.303053:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967140352:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765616.197880:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765616.197950:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765617.221818:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765617.221906:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765618.245802:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765618.245872:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765619.269812:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765619.269868:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765620.293885:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765620.293916:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765620.421889:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765620.421901:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765620.421908:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765620.421920:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2242 00000100:00080000:0.0:1775765620.421934:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765620.421942:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765620.421986:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967140480/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765620.422023:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765620.422028:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765620.422030:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765620.422036:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2242 00000100:00080000:0.0:1775765620.422042:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765620.422049:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765620.422107:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765620.422108:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967140480:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765620.422142:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967140608/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765620.422167:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765620.422172:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765620.422175:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765620.422180:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2242 00000100:00080000:0.0:1775765620.422199:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765620.422208:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:1.0:1775765620.422211:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:0.0:1775765620.422218:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765620.422259:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967140736/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765620.422273:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765620.422278:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765620.422280:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765620.422285:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2242 00000100:00080000:0.0:1775765620.422292:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765620.422297:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765620.422305:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765620.422307:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765620/real 1775765620] req@ffff8f9970a76300 x1862022967140480/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765620.422322:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967140864/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765620.422399:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967140480/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765620.422418:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765620.422426:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765620.422433:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967140480:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775765620.422492:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967140608:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775765620.422540:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765620.422566:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765620/real 1775765620] req@ffff8f9970a77100 x1862022967140608/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765620.422587:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967140608/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765620.422599:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765620.422603:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765620.422607:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967140608:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775765620.422616:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967140736:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765620.422700:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765620.422719:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765620/real 1775765620] req@ffff8f9970a75180 x1862022967140736/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765620.422734:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967140736/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765620.422817:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765620.422825:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765620.422831:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967140736:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775765620.422844:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967140864:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765620.422885:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765620.422907:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765620/real 1775765620] req@ffff8f9970a75500 x1862022967140864/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765620.422930:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967140864/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765620.422944:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765620.422950:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765620.422956:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967140864:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765621.317868:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765621.317929:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765622.341852:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765622.341961:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765623.365912:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765623.366013:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765624.389838:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765624.390034:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765625.413868:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765625.413974:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765625.541905:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765625.541942:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765625.541950:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765625.541967:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2247 00000100:00080000:0.0:1775765625.541988:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765625.542001:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765625.542066:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967140992/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765625.542108:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765625.542113:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765625.542115:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765625.542120:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2247 00000100:00080000:0.0:1775765625.542125:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765625.542130:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765625.542144:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765625.542159:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967141120/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765625.542172:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765625.542176:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765625.542178:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765625.542182:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2247 00000100:00080000:0.0:1775765625.542185:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775765625.542199:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967140992:192.168.202.146@tcp:250:kworker.0 00000800:00000100:1.0:1775765625.542287:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765625.542335:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765625/real 1775765625] req@ffff8f9970a77480 x1862022967140992/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765625.542356:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967140992/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765625.542368:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765625.542373:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765625.542377:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967140992:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775765625.542406:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765625.542421:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765625.542432:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967141120:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775765625.542459:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967141248/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765625.542468:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765625.542489:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765625/real 1775765625] req@ffff8f9970a77800 x1862022967141120/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765625.542491:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765625.542504:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765625.542510:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765625.542522:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2247 00000100:00100000:1.0:1775765625.542525:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967141120/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765625.542534:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765625.542536:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765625.542544:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765625.542550:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967141120:192.168.202.146@tcp:38:kworker.0 00000100:00080000:0.0:1775765625.542564:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765625.542585:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765625.542591:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967141248:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765625.542618:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967141376/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765625.542619:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765625.542662:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765625/real 1775765625] req@ffff8f9970a76a00 x1862022967141248/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765625.542678:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967141248/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765625.542710:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765625.542728:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765625.542732:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967141248:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775765625.542823:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967141376:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765625.542858:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765625.542877:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765625/real 1775765625] req@ffff8f9970a76d80 x1862022967141376/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765625.542928:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967141376/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765625.542937:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765625.542940:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765625.542944:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967141376:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765626.437874:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765626.437969:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765627.461809:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765627.461879:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765628.485805:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765628.485875:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765629.509861:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765629.509964:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765630.533853:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765630.534003:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765630.661901:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765630.661919:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765630.661927:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765630.661942:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2252 00000100:00080000:3.0:1775765630.661957:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765630.661967:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765630.662026:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967141504/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765630.662074:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765630.662079:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765630.662082:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765630.662087:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2252 00000100:00080000:3.0:1775765630.662091:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765630.662096:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765630.662106:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765630.662121:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967141632/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765630.662135:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765630.662139:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765630.662142:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765630.662146:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2252 00000100:00080000:3.0:1775765630.662151:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765630.662159:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765630.662166:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765630.662180:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967141760/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765630.662194:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765630.662199:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765630.662203:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775765630.662213:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967141504:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775765630.662213:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2252 00000100:00080000:3.0:1775765630.662237:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765630.662244:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765630.662250:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765630.662265:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967141888/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765630.662322:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765630.662389:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765630/real 1775765630] req@ffff8f9970edca80 x1862022967141504/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765630.662410:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967141504/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765630.662422:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765630.662429:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765630.662435:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967141504:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775765630.662442:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967141632:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775765630.662460:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765630.662469:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765630/real 1775765630] req@ffff8f9970edea00 x1862022967141632/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765630.662478:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967141632/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765630.662656:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765630.662659:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765630.662661:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967141632:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775765630.662689:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967141760:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765630.662714:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765630.662727:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765630/real 1775765630] req@ffff8f9970edf800 x1862022967141760/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765630.662780:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967141760/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765630.662786:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765630.662788:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765630.662791:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967141760:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775765630.662796:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967141888:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765630.662810:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765630.662817:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765630/real 1775765630] req@ffff8f9970edd880 x1862022967141888/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765630.662825:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967141888/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765630.662830:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765630.662832:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765630.662886:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967141888:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765631.557843:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765631.557947:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765632.581825:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765632.581950:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765633.605902:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765633.605928:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765634.629865:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765634.630041:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765635.653826:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765635.653880:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765635.781837:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765635.781845:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765635.781849:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765635.781858:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2257 00000100:00080000:1.0:1775765635.781867:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765635.781873:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765635.781909:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967142016/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765635.781933:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765635.781937:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765635.781938:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765635.781942:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2257 00000100:00080000:1.0:1775765635.781944:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765635.781947:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765635.781955:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765635.781965:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967142144/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765635.781974:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765635.781978:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765635.781979:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765635.781984:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967142016:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775765635.781985:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2257 00000100:00080000:1.0:1775765635.781990:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765635.781997:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765635.782001:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765635.782010:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967142272/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765635.782016:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765635.782019:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765635.782020:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765635.782023:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2257 00000100:00080000:1.0:1775765635.782025:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765635.782028:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765635.782032:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765635.782039:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967142400/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765635.782066:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765635.782121:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765635/real 1775765635] req@ffff8f994f2e6d80 x1862022967142016/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765635.782186:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967142016/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765635.782199:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765635.782204:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765635.782208:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967142016:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765635.782233:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967142144:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765635.782258:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765635.782272:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765635/real 1775765635] req@ffff8f994f2e4a80 x1862022967142144/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765635.782303:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967142144/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765635.782314:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765635.782318:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765635.782323:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967142144:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765635.782331:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967142272:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765635.782359:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765635.782389:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765635/real 1775765635] req@ffff8f994f2e7800 x1862022967142272/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765635.782403:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967142272/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765635.782438:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765635.782442:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765635.782446:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967142272:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765635.782453:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967142400:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765635.782478:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765635.782492:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765635/real 1775765635] req@ffff8f994f2e5180 x1862022967142400/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765635.782505:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967142400/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765635.782516:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765635.782519:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765635.782523:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967142400:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765636.677868:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765636.677921:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765637.701916:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765637.702012:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765638.725908:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765638.725957:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765639.749919:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765639.750041:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765640.773856:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765640.773978:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765640.901843:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765640.901851:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765640.901854:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765640.901863:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2262 00000100:00080000:3.0:1775765640.901873:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765640.901878:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765640.901910:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967142528/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765640.901931:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765640.901936:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765640.901937:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765640.901941:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2262 00000100:00080000:3.0:1775765640.901943:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765640.901945:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765640.901953:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765640.901961:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967142656/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765640.901970:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775765640.901972:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967142528:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775765640.901973:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765640.901974:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765640.901982:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2262 00000100:00080000:3.0:1775765640.901986:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765640.901988:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765640.901991:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765640.901998:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967142784/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765640.902004:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765640.902006:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765640.902007:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765640.902009:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2262 00000100:00080000:3.0:1775765640.902012:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765640.902014:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765640.902018:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765640.902024:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967142912/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765640.902049:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765640.902101:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765640/real 1775765640] req@ffff8f9970edf480 x1862022967142528/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765640.902121:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967142528/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765640.902160:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765640.902165:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765640.902170:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967142528:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765640.902191:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967142656:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765640.902230:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765640.902244:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765640/real 1775765640] req@ffff8f9970ede300 x1862022967142656/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765640.902269:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967142656/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765640.902280:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765640.902284:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765640.902303:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967142656:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765640.902310:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967142784:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765640.902328:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765640.902339:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765640/real 1775765640] req@ffff8f9970ede680 x1862022967142784/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765640.902351:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967142784/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765640.902360:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765640.902363:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765640.902375:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967142784:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765640.902394:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967142912:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765640.902413:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765640.902423:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765640/real 1775765640] req@ffff8f9970edc700 x1862022967142912/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765640.902433:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967142912/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765640.902441:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765640.902444:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765640.902447:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967142912:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765641.797889:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765641.797955:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765642.821895:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765642.821936:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775765643.845829:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765643.845891:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765644.869903:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765644.870006:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765645.893818:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765645.893918:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765646.021874:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765646.021882:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765646.021886:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765646.021894:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2267 00000100:00080000:0.0:1775765646.021904:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765646.021909:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765646.021942:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967143040/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765646.021967:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765646.021971:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765646.021973:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765646.021978:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2267 00000100:00080000:0.0:1775765646.021983:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765646.021986:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765646.021996:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765646.022011:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967143168/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765646.022015:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967143040:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775765646.022031:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765646.022035:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765646.022037:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765646.022042:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2267 00000100:00080000:0.0:1775765646.022047:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765646.022052:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765646.022058:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765646.022070:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967143296/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765646.022081:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765646.022084:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765646.022085:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765646.022088:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2267 00000100:00080000:0.0:1775765646.022091:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765646.022094:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765646.022098:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765646.022108:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967143424/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765646.022123:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967143168:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765646.022147:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967143296:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765646.022163:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967143424:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765646.917877:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765646.917934:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765647.941958:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765647.942028:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765648.965854:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765648.965893:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775765649.094015:0:2368:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:3.0:1775765649.094054:0:2368:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775765649.094070:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:3.0:1775765649.094104:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:3.0:1775765649.094111:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:3.0:1775765649.094118:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:2.0:1775765649.094172:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765646/real 1775765649] req@ffff8f9970a75c00 x1862022967143168/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765649.094206:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967143168/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765649.094221:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765649.094227:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765649.094233:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967143168:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765649.094256:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765646/real 1775765649] req@ffff8f9970a74700 x1862022967143296/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765649.094271:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967143296/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765649.094280:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765649.094285:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765649.094289:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967143296:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765649.094297:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765646/real 1775765649] req@ffff8f9970a76d80 x1862022967143424/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765649.094311:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967143424/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765649.094321:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765649.094326:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765649.094330:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967143424:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765649.094382:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765646/real 1775765649] req@ffff8f9970a74380 x1862022967143040/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765649.094399:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967143040/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765649.094431:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765649.094436:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765649.094441:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967143040:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765649.989883:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765649.989897:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765651.013873:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765651.013949:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765651.141926:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765651.141935:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765651.141939:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765651.141959:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2272 00000100:00080000:2.0:1775765651.141970:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765651.141977:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765651.142012:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2d80 x1862022967143552/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765651.142052:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765651.142055:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765651.142057:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765651.142061:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2272 00000100:00080000:2.0:1775765651.142064:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765651.142067:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765651.142100:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967143552:192.168.202.146@tcp:250:kworker.0 00000008:00100000:2.0:1775765651.142129:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765651.142152:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0e00 x1862022967143680/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765651.142164:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765651.142167:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765651.142170:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765651.142174:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2272 00000100:00080000:2.0:1775765651.142180:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1775765651.142185:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775765651.142190:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765651.142196:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765651.142211:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0700 x1862022967143808/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765651.142223:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765651.142227:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765651.142230:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775765651.142232:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765651/real 1775765651] req@ffff8f99703d2d80 x1862022967143552/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765651.142237:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2272 00000100:00080000:2.0:1775765651.142245:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765651.142260:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765651.142264:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2d80 x1862022967143552/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1775765651.142269:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1775765651.142288:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1775765651.142289:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1500 x1862022967143936/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765651.142297:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765651.142308:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967143552:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775765651.142338:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967143680:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775765651.142373:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765651.142428:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765651/real 1775765651] req@ffff8f99703d0e00 x1862022967143680/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765651.142444:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0e00 x1862022967143680/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765651.142452:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765651.142456:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765651.142459:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967143680:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775765651.142467:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967143808:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765651.142490:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765651.142502:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765651/real 1775765651] req@ffff8f99703d0700 x1862022967143808/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765651.142514:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0700 x1862022967143808/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765651.142523:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765651.142526:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765651.142529:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967143808:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775765651.142535:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967143936:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765651.142554:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765651.142565:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765651/real 1775765651] req@ffff8f99703d1500 x1862022967143936/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765651.142576:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1500 x1862022967143936/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765651.142583:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765651.142586:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765651.142589:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967143936:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765652.037833:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765652.037903:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765653.061849:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765653.061952:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765654.085893:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765654.086029:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765655.109865:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765655.109942:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765656.133892:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765656.133995:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765656.261907:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765656.261920:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765656.261925:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765656.261938:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2277 00000100:00080000:0.0:1775765656.261971:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765656.261980:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765656.262035:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967144064/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765656.262085:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765656.262089:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765656.262091:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765656.262094:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2277 00000100:00080000:0.0:1775765656.262097:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765656.262100:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765656.262109:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765656.262119:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967144192/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765656.262126:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765656.262129:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765656.262130:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765656.262133:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2277 00000100:00080000:0.0:1775765656.262135:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765656.262138:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765656.262141:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765656.262148:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967144320/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765656.262153:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765656.262156:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765656.262157:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765656.262160:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2277 00000100:00080000:0.0:1775765656.262162:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765656.262166:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765656.262168:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967144064:192.168.202.146@tcp:250:kworker.0 00000008:00100000:0.0:1775765656.262171:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765656.262181:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967144448/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765656.262274:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765656.262357:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765656/real 1775765656] req@ffff8f9970a76a00 x1862022967144064/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765656.262393:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967144064/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765656.262402:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765656.262406:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765656.262409:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967144064:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775765656.262418:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967144192:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775765656.262436:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765656.262445:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765656/real 1775765656] req@ffff8f9970a77800 x1862022967144192/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765656.262454:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967144192/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765656.262459:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765656.262462:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765656.262464:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967144192:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775765656.262468:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967144320:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765656.262478:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765656.262484:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765656/real 1775765656] req@ffff8f9970a77480 x1862022967144320/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765656.262491:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967144320/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765656.262496:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765656.262498:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765656.262500:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967144320:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775765656.262523:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967144448:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765656.262533:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765656.262539:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765656/real 1775765656] req@ffff8f9970a75500 x1862022967144448/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765656.262547:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967144448/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765656.262553:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765656.262554:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765656.262556:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967144448:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765657.157945:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765657.158038:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765658.181904:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765658.181987:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765659.205890:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765659.205894:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765660.229966:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765660.229972:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775765661.253886:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765661.253938:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765661.381847:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765661.381861:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765661.381867:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765661.381881:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2282 00000100:00080000:0.0:1775765661.381896:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765661.381906:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765661.381963:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967144576/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765661.382016:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765661.382024:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765661.382028:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765661.382037:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2282 00000100:00080000:0.0:1775765661.382044:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765661.382051:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765661.382065:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765661.382078:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967144576:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765661.382097:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967144704/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765661.382122:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765661.382130:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765661.382133:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1775765661.382167:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775765661.382210:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2282 00000100:00080000:0.0:1775765661.382219:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775765661.382223:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765661/real 1775765661] req@ffff8f9970a75180 x1862022967144576/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765661.382231:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765661.382249:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765661.382262:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967144576/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765661.382277:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967144832/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765661.382282:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765661.382324:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765661.382335:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967144576:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775765661.382341:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765661.382347:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765661.382351:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765661.382359:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2282 00000100:00080000:0.0:1775765661.382367:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775765661.382375:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967144704:192.168.202.146@tcp:38:kworker.0 00000100:00080000:0.0:1775765661.382376:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765661.382394:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765661.382417:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967144960/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765661.382427:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765661.382453:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765661/real 1775765661] req@ffff8f9970a77100 x1862022967144704/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765661.382550:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967144704/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765661.382561:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765661.382565:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765661.382570:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967144704:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775765661.382578:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967144832:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765661.382604:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765661.382617:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765661/real 1775765661] req@ffff8f9970a76300 x1862022967144832/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765661.382632:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967144832/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765661.382641:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765661.382645:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765661.382648:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967144832:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775765661.382666:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967144960:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765661.382685:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765661.382696:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765661/real 1775765661] req@ffff8f9970a75180 x1862022967144960/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765661.382710:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967144960/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765661.382782:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765661.382787:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765661.382791:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967144960:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765662.277949:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765662.278031:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765663.301915:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765663.302055:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765664.325954:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765664.326050:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775765665.349906:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765665.350032:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765666.373966:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765666.373993:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765666.501900:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765666.501911:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765666.501927:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765666.501956:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2288 00000100:00080000:1.0:1775765666.501970:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765666.501979:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765666.502028:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967145088/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765666.502083:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765666.502087:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765666.502089:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765666.502093:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2288 00000100:00080000:1.0:1775765666.502098:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765666.502102:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765666.502112:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765666.502126:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967145216/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765666.502178:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765666.502181:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765666.502184:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765666.502188:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2288 00000100:00080000:1.0:1775765666.502192:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765666.502195:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765666.502201:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765666.502216:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967145344/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765666.502233:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967145088:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775765666.502274:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765666.502282:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765666.502285:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765666.502294:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2288 00000100:00080000:1.0:1775765666.502302:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765666.502309:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765666.502319:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765666.502344:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967145472/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765666.502377:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765666.502483:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765666/real 1775765666] req@ffff8f994f2e4000 x1862022967145088/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765666.502515:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967145088/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765666.502530:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765666.502536:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765666.502540:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967145088:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765666.502549:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967145216:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765666.502576:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765666.502590:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765666/real 1775765666] req@ffff8f994f2e6a00 x1862022967145216/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765666.502722:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967145216/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765666.502733:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765666.502808:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765666.502815:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967145216:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765666.502894:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967145344:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765666.502943:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765666.502954:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765666/real 1775765666] req@ffff8f994f2e5f80 x1862022967145344/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765666.502964:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967145344/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765666.502971:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765666.502974:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765666.502976:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967145344:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765666.502981:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967145472:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765666.502995:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765666.503002:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765666/real 1775765666] req@ffff8f994f2e6680 x1862022967145472/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765666.503011:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967145472/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765666.503063:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765666.503066:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765666.503069:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967145472:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765667.397855:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765667.398069:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765668.421890:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765668.422045:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765669.445842:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765669.446064:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765670.469879:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765670.469975:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765671.494008:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765671.494017:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765671.622254:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765671.622270:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765671.622278:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765671.622299:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2293 00000100:00080000:0.0:1775765671.622316:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765671.622328:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765671.622397:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967145600/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765671.622504:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765671.622511:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765671.622516:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765671.622528:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2293 00000100:00080000:0.0:1775765671.622537:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765671.622545:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765671.622577:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765671.622579:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967145600:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765671.622609:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967145728/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765671.622646:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765671.622650:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765671.622653:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765671.622660:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2293 00000100:00080000:0.0:1775765671.622669:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765671.622677:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765671.622685:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765671.622708:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967145856/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765671.622721:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775765671.622743:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765671.622750:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765671.622754:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765671.622765:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2293 00000100:00100000:2.0:1775765671.622824:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765671/real 1775765671] req@ffff8f9970a75500 x1862022967145600/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765696 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765671.622852:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967145600/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765696 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765671.622868:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765671.622877:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765671.622878:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765671.622896:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1775765671.622897:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765671.622911:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967145600:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765671.622925:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967145984/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765671.622950:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967145728:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765671.622982:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765671.622994:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765671/real 1775765671] req@ffff8f9970a77480 x1862022967145728/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765696 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765671.623004:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967145728/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765696 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765671.623090:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765671.623093:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765671.623096:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967145728:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765671.623102:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967145856:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765671.623119:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765671.623128:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765671/real 1775765671] req@ffff8f9970a77800 x1862022967145856/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765696 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765671.623137:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967145856/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765696 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765671.623142:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765671.623144:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765671.623147:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967145856:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765671.623151:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967145984:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765671.623180:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765671.623215:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765671/real 1775765671] req@ffff8f9970a76a00 x1862022967145984/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765696 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765671.623230:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967145984/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765696 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765671.623240:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765671.623244:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765671.623285:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967145984:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765672.517905:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765672.518067:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765673.541871:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765673.542013:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765674.565902:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765674.565954:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765675.589857:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765675.589958:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765676.613842:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765676.613962:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765676.741915:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765676.741930:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765676.741954:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765676.741969:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2298 00000100:00080000:0.0:1775765676.741988:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765676.741998:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765676.742060:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967146112/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765676.742106:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765676.742111:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765676.742114:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765676.742120:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2298 00000100:00080000:0.0:1775765676.742125:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765676.742131:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765676.742192:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967146112:192.168.202.146@tcp:250:kworker.0 00000008:00100000:0.0:1775765676.742195:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765676.742221:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967146240/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765676.742236:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1775765676.742280:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765676.742325:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765676/real 1775765676] req@ffff8f9970a74380 x1862022967146112/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765676.742353:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967146112/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765676.742366:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765676.742371:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765676.742375:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967146112:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765676.742397:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967146240:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765676.742416:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765676.742425:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765676/real 1775765676] req@ffff8f9970a76d80 x1862022967146240/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765676.742435:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967146240/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765676.742441:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765676.742445:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765676.742452:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2298 00000100:00080000:2.0:1775765676.742456:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765676.742460:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765676.742461:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00080000:0.0:1775765676.742468:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765676.742478:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765676.742500:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967146368/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765676.742513:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765676.742515:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765676.742517:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765676.742519:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2298 00000100:00080000:0.0:1775765676.742521:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765676.742524:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765676.742526:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765676.742532:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967146496/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765676.742598:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967146240:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765676.742640:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967146368:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765676.742712:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765676.742788:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765676/real 1775765676] req@ffff8f9970a74700 x1862022967146368/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765676.742811:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967146368/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765676.742823:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765676.742828:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765676.742832:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967146368:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765676.742839:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967146496:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765676.742863:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765676.742874:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765676/real 1775765676] req@ffff8f9970a75c00 x1862022967146496/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765676.742887:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967146496/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765676.742896:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765676.742899:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765676.742902:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967146496:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765677.637915:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765677.637943:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765678.661890:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765678.661989:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775765679.685890:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765679.685999:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765680.709916:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765680.709952:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765681.733833:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765681.733885:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765681.861829:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765681.861838:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765681.861842:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765681.861851:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2303 00000100:00080000:1.0:1775765681.861860:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765681.861867:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765681.861907:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967146624/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765681.861935:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765681.861938:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765681.861939:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765681.861942:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2303 00000100:00080000:1.0:1775765681.861946:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765681.861949:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765681.861957:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765681.861968:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967146752/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765681.861977:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765681.861982:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1775765681.861984:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967146624:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775765681.862010:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765681.862016:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2303 00000100:00080000:1.0:1775765681.862021:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765681.862025:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765681.862032:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765681.862043:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765681.862046:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967146880/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765681.862055:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765681.862058:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765681.862060:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765681.862064:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2303 00000100:00080000:1.0:1775765681.862067:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765681.862069:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765681.862073:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765681.862081:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967147008/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765681.862117:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765681/real 1775765681] req@ffff8f994f2e7b80 x1862022967146624/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765681.862148:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967146624/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765681.862156:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765681.862159:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765681.862162:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967146624:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765681.862181:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967146752:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765681.862198:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765681.862207:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765681/real 1775765681] req@ffff8f994f2e6300 x1862022967146752/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765681.862215:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967146752/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765681.862232:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765681.862234:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765681.862236:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967146752:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765681.862240:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967146880:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765681.862291:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765681.862299:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765681/real 1775765681] req@ffff8f994f2e4e00 x1862022967146880/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765681.862314:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967146880/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765681.862319:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765681.862324:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765681.862326:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967146880:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765681.862331:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967147008:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765681.862342:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765681.862348:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765681/real 1775765681] req@ffff8f994f2e4700 x1862022967147008/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765681.862355:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967147008/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765681.862360:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765681.862361:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765681.862363:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967147008:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765682.757809:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765682.757867:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765683.781827:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765683.781883:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765684.805795:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765684.805848:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765685.829906:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765685.829928:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765686.853818:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765686.853896:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765686.981897:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765686.981907:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765686.981913:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765686.981925:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2308 00000100:00080000:1.0:1775765686.981939:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765686.981947:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765686.982104:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967147136/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765686.982142:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765686.982146:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765686.982148:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765686.982152:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2308 00000100:00080000:1.0:1775765686.982156:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765686.982160:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765686.982170:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765686.982182:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967147264/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765686.982191:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765686.982195:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765686.982196:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765686.982196:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967147136:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775765686.982200:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2308 00000100:00080000:1.0:1775765686.982202:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765686.982204:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765686.982208:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765686.982214:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967147392/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765686.982219:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765686.982221:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765686.982222:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765686.982254:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2308 00000100:00080000:1.0:1775765686.982256:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765686.982259:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765686.982262:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765686.982269:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967147520/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765686.982407:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765686.982459:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765686/real 1775765686] req@ffff8f994f2e4700 x1862022967147136/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765686.982483:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967147136/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765686.982496:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765686.982501:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765686.982506:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967147136:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765686.982545:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967147264:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765686.982581:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765686.982594:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765686/real 1775765686] req@ffff8f994f2e4e00 x1862022967147264/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765686.982610:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967147264/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765686.982620:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765686.982623:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765686.982627:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967147264:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765686.982636:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967147392:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765686.982662:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765686.982694:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765686/real 1775765686] req@ffff8f994f2e6300 x1862022967147392/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765686.982730:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967147392/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765686.982789:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765686.982793:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765686.982797:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967147392:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765686.982804:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967147520:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765686.982830:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765686.982846:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765686/real 1775765686] req@ffff8f994f2e7b80 x1862022967147520/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765686.982862:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967147520/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765686.982874:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765686.982882:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765686.982897:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967147520:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765687.877905:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765687.877989:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765688.901793:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765688.901873:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765689.925857:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765689.925920:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765690.949832:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765690.949899:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765691.973853:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765691.973971:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765692.101865:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765692.101879:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765692.101886:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765692.101899:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2313 00000100:00080000:3.0:1775765692.101916:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765692.101924:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765692.101979:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967147648/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765692.102023:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765692.102027:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765692.102029:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765692.102035:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2313 00000100:00080000:3.0:1775765692.102039:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765692.102045:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765692.102056:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967147648:192.168.202.146@tcp:250:kworker.0 00000008:00100000:3.0:1775765692.102058:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765692.102078:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967147776/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765692.102090:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765692.102094:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765692.102097:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765692.102101:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2313 00000100:00080000:3.0:1775765692.102132:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765692.102138:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765692.102145:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765692.102149:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765692.102165:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967147904/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765692.102177:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765692.102181:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765692.102184:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765692.102189:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2313 00000100:00080000:3.0:1775765692.102194:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775765692.102199:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765692/real 1775765692] req@ffff8f9970eded80 x1862022967147648/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765692.102202:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765692.102208:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765692.102220:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967148032/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765692.102350:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967147648/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765692.102360:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765692.102364:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765692.102368:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967147648:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765692.102397:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967147776:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765692.102452:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765692.102470:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765692/real 1775765692] req@ffff8f9970edc000 x1862022967147776/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765692.102478:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967147776/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765692.102484:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765692.102486:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765692.102488:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967147776:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765692.102492:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967147904:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765692.102504:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765692.102510:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765692/real 1775765692] req@ffff8f9970edce00 x1862022967147904/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765692.102517:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967147904/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765692.102541:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765692.102544:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765692.102546:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967147904:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765692.102550:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967148032:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765692.102562:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765692.102569:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765692/real 1775765692] req@ffff8f9970edd500 x1862022967148032/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765692.102576:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967148032/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765692.102581:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765692.102584:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765692.102586:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967148032:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765692.997862:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765692.997900:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765694.021831:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765694.021937:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765695.045791:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765695.045908:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765696.069801:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765696.069903:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765697.093824:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765697.093879:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765697.221875:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765697.221886:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765697.221891:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765697.221903:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2318 00000100:00080000:0.0:1775765697.221914:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765697.221921:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765697.221977:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967148160/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765697.222004:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765697.222007:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765697.222008:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765697.222012:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2318 00000100:00080000:0.0:1775765697.222015:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765697.222019:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765697.222026:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765697.222037:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967148288/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765697.222045:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775765697.222047:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967148160:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775765697.222048:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765697.222052:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765697.222057:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2318 00000100:00080000:0.0:1775765697.222059:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765697.222062:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765697.222065:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765697.222106:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765697.222109:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967148416/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765697.222117:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765697.222120:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765697.222122:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765697.222125:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2318 00000100:00080000:0.0:1775765697.222129:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765697.222132:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765697.222136:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765697.222145:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967148544/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765697.222166:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765697/real 1775765697] req@ffff8f9970a75c00 x1862022967148160/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765697.222187:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967148160/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765697.222220:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765697.222225:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765697.222231:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967148160:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765697.222252:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967148288:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765697.222271:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765697.222281:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765697/real 1775765697] req@ffff8f9970a74700 x1862022967148288/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765697.222289:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967148288/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765697.222294:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765697.222297:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765697.222299:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967148288:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765697.222302:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967148416:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765697.222315:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765697.222325:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765697/real 1775765697] req@ffff8f9970a76d80 x1862022967148416/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765697.222337:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967148416/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765697.222346:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765697.222349:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765697.222353:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967148416:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765697.222377:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967148544:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765697.222393:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765697.222401:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765697/real 1775765697] req@ffff8f9970a74380 x1862022967148544/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765697.222409:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967148544/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765697.222422:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765697.222424:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765697.222426:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967148544:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765698.117804:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765698.117853:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765699.141799:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765699.141877:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765700.165881:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765700.165952:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765701.189811:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765701.189865:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765702.213802:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765702.213918:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765702.341860:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765702.341875:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765702.341881:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765702.341907:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2323 00000100:00080000:1.0:1775765702.341922:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765702.341930:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765702.341979:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967148672/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765702.342014:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765702.342018:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765702.342020:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765702.342067:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967148672:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775765702.342089:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2323 00000100:00080000:1.0:1775765702.342095:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765702.342100:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765702.342112:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765702.342129:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967148800/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765702.342142:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775765702.342157:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765702.342161:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765702.342164:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765702.342170:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2323 00000100:00080000:1.0:1775765702.342175:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765702.342180:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765702.342185:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765702.342195:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765702/real 1775765702] req@ffff8f994f2e6680 x1862022967148672/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765702.342202:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967148928/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765702.342228:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775765702.342229:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967148672/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765702.342245:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765702.342253:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765702.342256:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765702.342261:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2323 00000100:00080000:2.0:1775765702.342265:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00080000:1.0:1775765702.342268:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775765702.342272:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967148672:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775765702.342277:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765702.342283:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765702.342295:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967149056/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765702.342323:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967148800:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765702.342350:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765702.342369:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765702/real 1775765702] req@ffff8f994f2e5f80 x1862022967148800/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765702.342380:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967148800/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765702.342386:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765702.342389:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765702.342392:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967148800:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765702.342399:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967148928:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765702.342412:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765702.342418:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765702/real 1775765702] req@ffff8f994f2e6a00 x1862022967148928/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765702.342425:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967148928/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765702.342430:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765702.342432:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765702.342434:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967148928:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765702.342443:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967149056:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765702.342454:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765702.342463:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765702/real 1775765702] req@ffff8f994f2e4000 x1862022967149056/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765702.342493:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967149056/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765702.342502:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765702.342506:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765702.342509:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967149056:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765703.237841:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765703.237928:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765704.261825:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765704.261892:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765705.285852:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765705.285918:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765706.309798:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765706.309906:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765707.333917:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765707.333952:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765707.461878:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765707.461886:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765707.461890:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765707.461898:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2329 00000100:00080000:3.0:1775765707.461908:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765707.461914:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765707.461948:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddc00 x1862022967149184/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765707.461971:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765707.461974:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765707.461975:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765707.461979:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2329 00000100:00080000:3.0:1775765707.461981:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765707.461984:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765707.461992:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765707.462003:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf100 x1862022967149312/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765707.462004:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967149184:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775765707.462023:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765707.462026:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765707.462027:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765707.462030:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2329 00000100:00080000:3.0:1775765707.462033:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765707.462035:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765707.462039:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765707.462050:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967149440/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765707.462061:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765707.462063:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765707.462065:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1775765707.462068:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775765707.462072:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2329 00000100:00080000:3.0:1775765707.462075:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765707.462078:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765707.462081:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765707.462089:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967149568/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765707.462115:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765707/real 1775765707] req@ffff8f9970eddc00 x1862022967149184/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765707.462132:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddc00 x1862022967149184/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765707.462140:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765707.462143:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765707.462146:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967149184:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765707.462163:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967149312:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765707.462178:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765707.462186:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765707/real 1775765707] req@ffff8f9970edf100 x1862022967149312/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765707.462225:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf100 x1862022967149312/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765707.462231:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765707.462233:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765707.462235:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967149312:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765707.462239:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967149440:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765707.462252:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765707.462259:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765707/real 1775765707] req@ffff8f9970edd500 x1862022967149440/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765707.462266:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967149440/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765707.462271:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765707.462273:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765707.462275:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967149440:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765707.462279:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967149568:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765707.462288:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765707.462293:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765707/real 1775765707] req@ffff8f9970edce00 x1862022967149568/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765707.462304:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967149568/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765707.462334:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765707.462338:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765707.462342:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967149568:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765708.357891:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765708.357952:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765709.381926:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765709.381977:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765710.405809:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765710.405962:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765711.429836:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765711.429953:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765712.453893:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765712.453979:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765712.581892:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765712.581903:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765712.581908:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765712.581919:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2334 00000100:00080000:3.0:1775765712.581932:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765712.581940:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765712.581986:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967149696/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765712.582032:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765712.582035:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765712.582036:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765712.582040:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2334 00000100:00080000:3.0:1775765712.582044:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765712.582048:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765712.582077:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765712.582086:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967149696:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765712.582096:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967149824/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765712.582107:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765712.582111:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765712.582112:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765712.582119:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2334 00000100:00080000:3.0:1775765712.582122:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765712.582125:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765712.582131:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765712.582143:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967149952/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765712.582150:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765712.582153:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765712.582156:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765712.582160:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2334 00000100:00080000:3.0:1775765712.582163:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765712.582166:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765712.582169:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765712.582177:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967150080/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765712.582232:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967149824:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765712.582274:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967149952:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765712.582307:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967150080:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765713.477875:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765713.478038:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765714.502006:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765714.502036:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765715.525949:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765715.526079:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1775765715.654224:0:2367:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:0.0:1775765715.654305:0:2367:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1775765715.654332:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775765715.654402:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775765715.654414:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775765715.654424:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:2.0:1775765715.654468:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765712/real 1775765715] req@ffff8f9970eded80 x1862022967149824/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765715.654503:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967149824/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765715.654519:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765715.654527:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765715.654532:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967149824:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765715.654835:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765712/real 1775765715] req@ffff8f9970edc700 x1862022967149952/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765715.654873:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967149952/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765715.654885:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765715.654890:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765715.654896:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967149952:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765715.654904:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765712/real 1775765715] req@ffff8f9970ede680 x1862022967150080/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765715.654919:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967150080/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765715.654929:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765715.654933:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765715.654937:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967150080:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765715.654946:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765712/real 1775765715] req@ffff8f9970edc000 x1862022967149696/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765715.654960:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967149696/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765715.654970:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765715.654975:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765715.654978:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967149696:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765716.549831:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765716.549930:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765717.573893:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765717.573987:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765717.701870:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765717.701879:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765717.701883:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765717.701892:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2339 00000100:00080000:3.0:1775765717.701902:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765717.701908:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765717.701940:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967150208/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765717.701966:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765717.701969:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765717.701970:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765717.701974:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2339 00000100:00080000:3.0:1775765717.701977:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765717.702009:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765717.702018:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765717.702031:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967150208:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765717.702031:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967150336/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765717.702047:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765717.702049:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765717.702051:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765717.702054:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2339 00000100:00080000:3.0:1775765717.702072:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765717.702075:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765717.702081:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765717.702094:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967150464/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765717.702112:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765717.702116:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765717.702120:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1775765717.702121:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775765717.702129:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2339 00000100:00080000:3.0:1775765717.702135:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765717.702140:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765717.702146:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765717.702160:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967150592/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765717.702187:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765717/real 1775765717] req@ffff8f9970ede300 x1862022967150208/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765717.702209:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967150208/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765717.702220:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765717.702225:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765717.702230:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967150208:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765717.702266:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967150336:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765717.702291:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765717.702304:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765717/real 1775765717] req@ffff8f9970edf480 x1862022967150336/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765717.702317:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967150336/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765717.702325:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765717.702329:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765717.702333:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967150336:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765717.702340:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967150464:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765717.702421:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765717.702434:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765717/real 1775765717] req@ffff8f9970edd880 x1862022967150464/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765717.702447:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967150464/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765717.702456:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765717.702461:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765717.702464:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967150464:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765717.702471:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967150592:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765717.702492:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765717.702504:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765717/real 1775765717] req@ffff8f9970edf800 x1862022967150592/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765717.702517:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967150592/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765717.702525:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765717.702529:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765717.702533:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967150592:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765718.597860:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765718.597968:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765719.621836:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765719.621947:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765720.645903:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765720.645998:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765721.669915:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765721.669927:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775765722.693877:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765722.693989:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765722.821840:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765722.821863:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765722.821869:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765722.821881:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2344 00000100:00080000:3.0:1775765722.821895:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765722.821902:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765722.821936:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967150720/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765722.821967:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765722.821971:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765722.821972:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765722.821976:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2344 00000100:00080000:3.0:1775765722.821979:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765722.821982:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765722.821992:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765722.822010:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967150848/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765722.822022:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967150720:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775765722.822023:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765722.822033:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765722.822036:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765722.822041:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2344 00000100:00080000:3.0:1775765722.822044:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765722.822047:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765722.822051:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765722.822062:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967150976/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765722.822071:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765722.822074:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765722.822076:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765722.822079:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2344 00000100:00080000:3.0:1775765722.822083:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765722.822087:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765722.822092:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775765722.822094:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765722.822105:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967151104/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765722.822140:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765722/real 1775765722] req@ffff8f9970edea00 x1862022967150720/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765722.822225:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967150720/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765722.822236:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765722.822240:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765722.822244:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967150720:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775765722.822266:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967150848:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775765722.822288:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765722.822300:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765722/real 1775765722] req@ffff8f9970edca80 x1862022967150848/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765722.822409:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967150848/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765722.822426:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765722.822433:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765722.822442:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967150848:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775765722.822457:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967150976:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765722.822505:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765722.822543:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765722/real 1775765722] req@ffff8f9970edc380 x1862022967150976/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765722.822555:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967150976/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765722.822562:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765722.822565:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765722.822567:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967150976:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775765722.822583:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967151104:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765722.822600:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765722.822613:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765722/real 1775765722] req@ffff8f9970edfb80 x1862022967151104/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765722.822625:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967151104/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765722.822653:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765722.822656:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765722.822659:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967151104:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765723.717802:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765723.717866:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765724.741823:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765724.741885:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765725.765792:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765725.765984:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765726.789834:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765726.789982:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765727.813832:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765727.813949:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765727.941861:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765727.941872:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765727.941880:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765727.941893:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2349 00000100:00080000:3.0:1775765727.941908:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765727.941916:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765727.941974:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddf80 x1862022967151232/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765727.942027:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765727.942032:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765727.942035:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765727.942040:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2349 00000100:00080000:3.0:1775765727.942044:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765727.942048:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765727.942059:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765727.942073:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd180 x1862022967151360/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765727.942085:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765727.942090:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765727.942092:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765727.942096:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2349 00000100:00080000:3.0:1775765727.942099:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765727.942103:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765727.942108:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765727.942120:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967151488/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765727.942141:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765727.942145:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765727.942148:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765727.942153:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2349 00000100:00080000:3.0:1775765727.942156:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765727.942160:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765727.942175:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765727.942182:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967151232:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765727.942193:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967151616/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765727.942281:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765727.942376:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765727/real 1775765727] req@ffff8f9970eddf80 x1862022967151232/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765727.942400:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddf80 x1862022967151232/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765727.942412:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765727.942418:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765727.942425:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967151232:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775765727.942435:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967151360:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775765727.942461:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765727.942474:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765727/real 1775765727] req@ffff8f9970edd180 x1862022967151360/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765727.942488:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd180 x1862022967151360/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765727.942556:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765727.942561:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765727.942565:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967151360:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775765727.942572:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967151488:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765727.942594:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765727.942606:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765727/real 1775765727] req@ffff8f9970edfb80 x1862022967151488/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765727.942619:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967151488/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765727.942628:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765727.942631:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765727.942634:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967151488:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775765727.942700:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967151616:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765727.942722:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765727.942734:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765727/real 1775765727] req@ffff8f9970edc380 x1862022967151616/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765727.942783:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967151616/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765727.942793:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765727.942797:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765727.942821:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967151616:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775765728.837945:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765728.837947:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775765729.861896:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765729.861996:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765730.885851:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765730.885945:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765731.909798:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765731.909886:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765732.933805:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765732.933967:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765733.061834:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765733.061842:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765733.061846:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765733.061854:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2354 00000100:00080000:0.0:1775765733.061876:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765733.061881:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765733.061913:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967151744/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765733.061946:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765733.061949:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765733.061955:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765733.061958:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2354 00000100:00080000:0.0:1775765733.061961:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765733.061963:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765733.061971:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765733.061982:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967151872/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765733.061990:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765733.061992:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765733.061994:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775765733.061997:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967151744:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775765733.062041:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2354 00000100:00080000:0.0:1775765733.062057:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765733.062059:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765733.062063:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775765733.062067:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765733.062076:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967152000/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765733.062084:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765733.062087:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765733.062088:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765733.062091:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2354 00000100:00080000:0.0:1775765733.062093:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765733.062096:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765733.062099:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765733.062104:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765733/real 1775765733] req@ffff8f9970a76a00 x1862022967151744/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765733.062109:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967152128/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765733.062129:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967151744/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765733.062140:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765733.062143:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765733.062146:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967151744:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775765733.062164:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967151872:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775765733.062179:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765733.062186:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765733/real 1775765733] req@ffff8f9970a77800 x1862022967151872/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765733.062194:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967151872/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765733.062200:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765733.062202:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765733.062204:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967151872:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775765733.062228:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967152000:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765733.062241:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765733.062248:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765733/real 1775765733] req@ffff8f9970a77480 x1862022967152000/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765733.062256:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967152000/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765733.062261:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765733.062263:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765733.062265:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967152000:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775765733.062269:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967152128:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765733.062279:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765733.062292:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765733/real 1775765733] req@ffff8f9970a75500 x1862022967152128/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765733.062300:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967152128/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765733.062305:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765733.062307:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765733.062309:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967152128:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765733.957795:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765733.957910:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765734.981891:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765734.982001:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765736.005882:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765736.005894:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765737.029844:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765737.029934:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765738.053816:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765738.053921:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765738.181997:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765738.182014:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765738.182023:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765738.182060:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2359 00000100:00080000:0.0:1775765738.182099:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765738.182113:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765738.182182:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967152256/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765738.183060:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765738.183065:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765738.183068:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765738.183077:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2359 00000100:00080000:0.0:1775765738.183090:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765738.183100:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765738.183116:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765738.183141:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967152256:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765738.183160:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967152384/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765738.183177:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765738.183181:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765738.183183:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765738.183189:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2359 00000100:00080000:0.0:1775765738.183194:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765738.183199:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765738.183207:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765738.183222:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967152512/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765738.183236:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765738.183240:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765738.183242:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765738.183247:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2359 00000100:00080000:0.0:1775765738.183274:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765738.183280:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765738.183286:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765738.183302:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967152640/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765738.183385:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765738.183443:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765738/real 1775765738] req@ffff8f9970a75180 x1862022967152256/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765738.183470:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967152256/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765738.183486:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765738.183492:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765738.183498:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967152256:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775765738.183532:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967152384:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775765738.183584:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765738.183603:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765738/real 1775765738] req@ffff8f9970a76300 x1862022967152384/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765738.183624:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967152384/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765738.183632:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765738.183636:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765738.183639:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967152384:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775765738.183648:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967152512:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765738.183694:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765738.183712:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765738/real 1775765738] req@ffff8f9970a77100 x1862022967152512/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765738.183779:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967152512/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765738.183790:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765738.183793:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765738.183796:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967152512:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775765738.183802:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967152640:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765738.183823:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765738.183832:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765738/real 1775765738] req@ffff8f9970a75500 x1862022967152640/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765738.183842:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967152640/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765738.183849:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765738.183851:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765738.183853:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967152640:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765739.077880:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765739.077965:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765740.101881:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765740.101971:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765741.125913:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765741.126173:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765742.149843:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765742.149909:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765743.173865:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765743.173906:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765743.301842:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765743.301850:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765743.301855:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765743.301863:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2364 00000100:00080000:0.0:1775765743.301873:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765743.301879:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765743.301911:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967152768/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765743.301935:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765743.301938:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765743.301939:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765743.301942:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2364 00000100:00080000:0.0:1775765743.301944:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765743.301946:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765743.301954:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765743.301962:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967152896/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765743.301986:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967152768:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775765743.302002:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765743.302004:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765743.302005:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765743.302009:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2364 00000100:00080000:0.0:1775765743.302011:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765743.302014:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765743.302017:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765743.302026:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967153024/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765743.302032:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765743.302035:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765743.302036:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765743.302038:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2364 00000100:00080000:0.0:1775765743.302041:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765743.302043:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765743.302046:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775765743.302050:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765743.302056:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967153152/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765743.302089:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765743/real 1775765743] req@ffff8f9970a77480 x1862022967152768/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765743.302127:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967152768/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765743.302134:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765743.302137:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765743.302140:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967152768:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775765743.302163:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967152896:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775765743.302186:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765743.302200:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765743/real 1775765743] req@ffff8f9970a77800 x1862022967152896/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765743.302214:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967152896/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765743.302223:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765743.302227:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765743.302229:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967152896:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775765743.302233:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967153024:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765743.302249:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765743.302256:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765743/real 1775765743] req@ffff8f9970a76a00 x1862022967153024/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765743.302264:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967153024/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765743.302287:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765743.302289:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765743.302291:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967153024:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775765743.302295:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967153152:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765743.302307:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765743.302313:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765743/real 1775765743] req@ffff8f9970a74380 x1862022967153152/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765743.302320:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967153152/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765743.302326:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765743.302328:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765743.302330:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967153152:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765744.197831:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765744.197929:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765745.221846:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765745.222050:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765746.245935:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765746.246058:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765747.269856:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765747.269969:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765748.293938:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765748.294050:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765748.421902:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765748.421911:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765748.421915:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765748.421924:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2370 00000100:00080000:1.0:1775765748.421935:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765748.421940:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765748.421973:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967153280/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765748.422037:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765748.422040:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765748.422041:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765748.422044:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2370 00000100:00080000:1.0:1775765748.422047:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765748.422049:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765748.422056:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765748.422065:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967153408/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765748.422072:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765748.422074:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765748.422076:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765748.422080:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2370 00000100:00080000:1.0:1775765748.422083:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765748.422088:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765748.422094:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765748.422107:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967153536/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765748.422124:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765748.422134:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1775765748.422137:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967153280:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775765748.422139:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765748.422171:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2370 00000100:00080000:1.0:1775765748.422193:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765748.422199:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765748.422205:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765748.422243:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967153664/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765748.422294:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765748.422393:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765748/real 1775765748] req@ffff8f994f2e5180 x1862022967153280/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765748.422425:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967153280/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765748.422694:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765748.422699:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765748.422703:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967153280:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765748.422712:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967153408:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765748.422801:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765748.422825:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765748/real 1775765748] req@ffff8f994f2e7800 x1862022967153408/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765748.422838:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967153408/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765748.422844:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765748.422847:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765748.422865:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967153408:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765748.422894:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967153536:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765748.422910:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765748.422918:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765748/real 1775765748] req@ffff8f994f2e4a80 x1862022967153536/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765748.422925:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967153536/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765748.422931:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765748.422933:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765748.422935:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967153536:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765748.422954:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967153664:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765748.422967:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765748.422973:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765748/real 1775765748] req@ffff8f994f2e6d80 x1862022967153664/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765748.422981:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967153664/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765748.422986:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765748.422988:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765748.422990:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967153664:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765749.317838:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765749.317973:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765750.341984:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765750.342067:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765751.365852:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765751.366051:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765752.389875:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765752.390027:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765753.413816:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765753.413893:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765753.541864:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765753.541875:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765753.541880:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765753.541890:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2375 00000100:00080000:1.0:1775765753.541901:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765753.541908:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765753.541948:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967153792/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765753.541975:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765753.541979:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765753.541981:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765753.541986:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2375 00000100:00080000:1.0:1775765753.541990:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765753.541995:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765753.542005:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765753.542011:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967153792:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775765753.542023:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967153920/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765753.542068:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775765753.542082:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765753.542086:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765753.542088:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765753.542093:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2375 00000100:00080000:1.0:1775765753.542098:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765753.542103:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765753.542105:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765753/real 1775765753] req@ffff8f994f2e6d80 x1862022967153792/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1775765753.542111:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765753.542122:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967153792/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765753.542129:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967154048/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765753.542131:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765753.542139:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765753.542146:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967153792:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775765753.542152:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765753.542155:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765753.542157:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765753.542161:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2375 00000100:00080000:1.0:1775765753.542166:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765753.542172:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765753.542178:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765753.542187:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967153920:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775765753.542193:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967154176/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765753.542208:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765753.542220:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765753/real 1775765753] req@ffff8f994f2e4a80 x1862022967153920/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765753.542229:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967153920/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765753.542235:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765753.542237:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765753.542239:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967153920:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765753.542243:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967154048:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765753.542254:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765753.542260:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765753/real 1775765753] req@ffff8f994f2e7800 x1862022967154048/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765753.542267:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967154048/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765753.542272:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765753.542274:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765753.542275:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967154048:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765753.542283:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967154176:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765753.542293:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765753.542298:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765753/real 1775765753] req@ffff8f994f2e5180 x1862022967154176/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765753.542318:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967154176/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765753.542323:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765753.542326:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765753.542328:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967154176:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765754.437867:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765754.438070:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765755.461898:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765755.461988:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765756.485831:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765756.485905:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765757.509956:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765757.509972:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765758.533861:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765758.533946:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765758.661868:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765758.661875:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765758.661880:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765758.661890:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2380 00000100:00080000:2.0:1775765758.661901:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765758.661907:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765758.661943:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1180 x1862022967154304/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765758.661971:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765758.661976:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765758.661979:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765758.661984:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2380 00000100:00080000:2.0:1775765758.661987:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775765758.662021:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967154304:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775765758.662033:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765758.662043:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765758.662057:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3100 x1862022967154432/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765758.662068:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765758.662072:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765758.662076:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765758.662083:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2380 00000100:00080000:2.0:1775765758.662089:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765758.662093:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765758.662099:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765758.662110:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3480 x1862022967154560/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765758.662116:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775765758.662128:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765758.662132:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765758.662134:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765758.662140:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2380 00000100:00080000:2.0:1775765758.662144:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765758.662146:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765758.662150:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765758.662159:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2300 x1862022967154688/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765758.662171:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765758/real 1775765758] req@ffff8f99703d1180 x1862022967154304/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765758.662205:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1180 x1862022967154304/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765758.662216:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765758.662220:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765758.662223:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967154304:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775765758.662244:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967154432:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775765758.662276:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765758.662290:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765758/real 1775765758] req@ffff8f99703d3100 x1862022967154432/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765758.662304:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3100 x1862022967154432/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765758.662311:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765758.662315:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765758.662318:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967154432:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775765758.662323:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967154560:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765758.662337:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765758.662346:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765758/real 1775765758] req@ffff8f99703d3480 x1862022967154560/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765758.662408:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3480 x1862022967154560/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765758.662420:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765758.662425:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765758.662430:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967154560:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775765758.662440:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967154688:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765758.662464:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765758.662478:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765758/real 1775765758] req@ffff8f99703d2300 x1862022967154688/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765758.662489:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2300 x1862022967154688/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765758.662497:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765758.662500:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765758.662503:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967154688:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765759.557874:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765759.557933:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765760.581813:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765760.581894:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765761.605891:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765761.605970:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765762.629832:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765762.629976:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765763.653840:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765763.653964:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765763.781881:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765763.781890:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765763.781894:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765763.781902:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2385 00000100:00080000:0.0:1775765763.781911:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765763.781916:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765763.781948:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967154816/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765763.781973:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765763.781976:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765763.781977:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765763.781980:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2385 00000100:00080000:0.0:1775765763.781982:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765763.781984:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765763.781992:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765763.782000:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967154944/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765763.782008:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765763.782011:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765763.782012:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765763.782015:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2385 00000100:00080000:0.0:1775765763.782017:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775765763.782021:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967154816:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775765763.782057:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765763.782061:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765763.782069:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967155072/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765763.782076:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1775765763.782080:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775765763.782082:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765763.782084:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765763.782088:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2385 00000100:00080000:0.0:1775765763.782091:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765763.782094:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765763.782098:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765763.782106:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967155200/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765763.782121:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765763/real 1775765763] req@ffff8f9970a76d80 x1862022967154816/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765763.782158:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967154816/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765763.782166:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765763.782169:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765763.782172:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967154816:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775765763.782191:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967154944:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775765763.782208:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765763.782216:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765763/real 1775765763] req@ffff8f9970a74700 x1862022967154944/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765763.782224:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967154944/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765763.782230:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765763.782232:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765763.782234:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967154944:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775765763.782238:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967155072:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765763.782249:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765763.782255:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765763/real 1775765763] req@ffff8f9970a75c00 x1862022967155072/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765763.782265:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967155072/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765763.782295:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765763.782299:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765763.782303:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967155072:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775765763.782341:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967155200:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765763.782365:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765763.782374:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765763/real 1775765763] req@ffff8f9970a74380 x1862022967155200/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765763.782382:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967155200/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765763.782387:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765763.782389:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765763.782391:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967155200:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775765764.677869:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765764.677881:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775765765.701842:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765765.701868:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765766.725826:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765766.725932:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765767.749877:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765767.749929:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765768.773873:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765768.773944:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765768.901999:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765768.902029:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765768.902036:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765768.902051:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2390 00000100:00080000:0.0:1775765768.902065:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765768.902074:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765768.902167:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967155328/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765768.902196:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775765768.902246:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967155328:192.168.202.146@tcp:250:kworker.0 00000800:00000100:1.0:1775765768.902332:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765768.902368:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765768/real 1775765768] req@ffff8f9970a76a00 x1862022967155328/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765768.902383:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967155328/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765768.902396:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765768.902399:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765768.902406:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2390 00000100:00080000:0.0:1775765768.902412:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765768.902421:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00080000:1.0:1775765768.902422:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765768.902427:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765768.902431:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967155328:192.168.202.146@tcp:250:kworker.0 00000008:00100000:0.0:1775765768.902432:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765768.902457:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967155456/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765768.902478:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765768.902482:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765768.902484:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765768.902487:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2390 00000100:00080000:0.0:1775765768.902490:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775765768.902492:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967155456:192.168.202.146@tcp:38:kworker.0 00000100:00080000:0.0:1775765768.902496:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765768.902502:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765768.902513:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967155584/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765768.902521:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775765768.902522:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765768.902526:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765768.902528:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765768.902532:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2390 00000100:00100000:1.0:1775765768.902535:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765768/real 1775765768] req@ffff8f9970a77800 x1862022967155456/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765768.902536:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765768.902547:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765768.902550:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967155456/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765768.902557:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765768.902560:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000008:00100000:0.0:1775765768.902565:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765768.902565:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967155456:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775765768.902582:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967155712/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765768.902583:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967155584:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765768.902616:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765768.902630:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765768/real 1775765768] req@ffff8f9970a77480 x1862022967155584/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765768.902644:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967155584/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765768.902653:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765768.902655:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765768.902657:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967155584:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775765768.902681:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967155712:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765768.902697:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765768.902704:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765768/real 1775765768] req@ffff8f9970a75500 x1862022967155712/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765768.902712:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967155712/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765768.902718:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765768.902720:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765768.902722:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967155712:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765769.797821:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765769.797848:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765770.821867:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765770.821939:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765771.845946:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765771.845946:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775765772.869849:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765772.869950:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765773.893836:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765773.893949:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765774.021867:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765774.021879:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765774.021884:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765774.021896:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2395 00000100:00080000:1.0:1775765774.021910:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765774.021920:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765774.021961:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967155840/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765774.021997:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765774.022002:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765774.022004:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765774.022041:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967155840:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775765774.022053:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2395 00000100:00080000:1.0:1775765774.022062:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765774.022068:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765774.022080:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765774.022098:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967155968/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765774.022130:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765774.022135:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765774.022138:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1775765774.022144:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775765774.022150:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2395 00000100:00080000:1.0:1775765774.022157:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765774.022173:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765774.022180:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765774.022193:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967156096/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765774.022235:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765774.022239:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765774.022240:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765774.022244:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765774/real 1775765774] req@ffff8f994f2e4000 x1862022967155840/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765774.022248:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2395 00000100:00080000:1.0:1775765774.022253:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765774.022261:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765774.022272:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765774.022283:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967155840/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765774.022288:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967156224/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765774.022296:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765774.022300:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765774.022304:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967155840:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765774.022341:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967155968:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765774.022363:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765774.022373:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765774/real 1775765774] req@ffff8f994f2e6a00 x1862022967155968/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765774.022410:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967155968/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765774.022417:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765774.022420:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765774.022423:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967155968:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765774.022428:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967156096:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765774.022443:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765774.022451:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765774/real 1775765774] req@ffff8f994f2e5f80 x1862022967156096/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765774.022460:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967156096/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765774.022466:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765774.022468:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765774.022470:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967156096:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765774.022475:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967156224:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765774.022486:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765774.022493:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765774/real 1775765774] req@ffff8f994f2e6680 x1862022967156224/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765774.022502:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967156224/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765774.022543:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765774.022546:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765774.022549:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967156224:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765774.917857:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765774.917910:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765775.941840:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765775.941921:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765776.965832:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765776.965892:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765777.989841:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765777.989917:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765779.014005:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765779.014016:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765779.141885:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765779.141894:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765779.141898:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765779.141906:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2400 00000100:00080000:1.0:1775765779.141917:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765779.141923:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765779.141957:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967156352/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765779.141982:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765779.141985:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765779.141986:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765779.141990:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2400 00000100:00080000:1.0:1775765779.141994:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765779.141997:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765779.142008:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765779.142022:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967156480/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765779.142034:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765779.142038:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765779.142040:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765779.142053:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967156352:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775765779.142086:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2400 00000100:00080000:1.0:1775765779.142092:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765779.142099:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765779.142106:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765779.142120:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967156608/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765779.142138:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765779.142144:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765779.142146:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765779.142151:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2400 00000100:00080000:1.0:1775765779.142156:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765779.142187:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765779.142195:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765779.142207:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967156480:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775765779.142218:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967156736/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765779.142249:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967156608:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765779.142272:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967156736:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765780.037850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765780.038043:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765781.061861:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765781.061962:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765782.085878:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765782.085998:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1775765782.214018:0:2369:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:0.0:1775765782.214085:0:2369:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1775765782.214102:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775765782.214175:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775765782.214190:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775765782.214205:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:2.0:1775765782.214247:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765779/real 1775765782] req@ffff8f994f2e4700 x1862022967156736/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765782.214289:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967156736/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765782.214304:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765782.214311:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765782.214315:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967156736:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765782.214512:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765779/real 1775765782] req@ffff8f994f2e6300 x1862022967156480/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765782.214528:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967156480/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765782.214536:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765782.214540:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765782.214543:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967156480:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765782.214551:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765779/real 1775765782] req@ffff8f994f2e4e00 x1862022967156608/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765782.214561:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967156608/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765782.214568:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765782.214570:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765782.214573:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967156608:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765782.214581:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765779/real 1775765782] req@ffff8f994f2e7b80 x1862022967156352/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765782.214616:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967156352/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765782.214623:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765782.214626:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765782.214629:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967156352:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765783.109830:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765783.109894:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765784.133859:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765784.134014:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765784.261823:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765784.261839:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765784.261844:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765784.261852:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2405 00000100:00080000:1.0:1775765784.261861:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765784.261868:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765784.261935:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967156864/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765784.261970:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765784.261974:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765784.261976:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765784.261981:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2405 00000100:00080000:1.0:1775765784.261986:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765784.261989:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765784.261999:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765784.262016:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967156992/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765784.262026:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967156864:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775765784.262031:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765784.262038:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765784.262040:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765784.262046:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2405 00000100:00080000:1.0:1775765784.262051:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765784.262055:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765784.262060:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765784.262073:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967157120/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765784.262083:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765784.262087:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765784.262089:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1775765784.262105:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775765784.262128:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2405 00000100:00080000:1.0:1775765784.262134:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765784.262137:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765784.262145:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765784.262158:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967157248/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765784.262187:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765784/real 1775765784] req@ffff8f994f2e7b80 x1862022967156864/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765784.262209:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967156864/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765784.262221:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765784.262225:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765784.262230:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967156864:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765784.262256:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967156992:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765784.262280:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765784.262293:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765784/real 1775765784] req@ffff8f994f2e4e00 x1862022967156992/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765784.262305:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967156992/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765784.262314:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765784.262318:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765784.262322:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967156992:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765784.262328:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967157120:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765784.262375:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765784.262387:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765784/real 1775765784] req@ffff8f994f2e6300 x1862022967157120/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765784.262398:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967157120/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765784.262406:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765784.262409:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765784.262412:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967157120:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765784.262417:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967157248:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765784.262435:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765784.262444:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765784/real 1775765784] req@ffff8f994f2e4700 x1862022967157248/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765784.262454:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967157248/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765784.262462:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765784.262465:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765784.262467:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967157248:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765785.157865:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765785.157969:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765786.181900:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765786.181915:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775765787.205870:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765787.205947:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765788.229877:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765788.229988:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765789.253891:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765789.254022:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765789.381829:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765789.381837:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765789.381842:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765789.381853:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2410 00000100:00080000:1.0:1775765789.381865:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765789.381872:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765789.381924:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967157376/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765789.381952:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765789.381955:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765789.381957:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765789.381960:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2410 00000100:00080000:1.0:1775765789.381964:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765789.381978:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765789.381988:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765789.382001:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967157376:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775765789.382038:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967157504/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765789.382050:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765789.382053:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765789.382055:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765789.382058:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2410 00000800:00000100:2.0:1775765789.382078:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775765789.382079:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765789.382086:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765789.382092:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765789.382113:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967157632/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765789.382121:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765789.382123:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765789.382125:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765789.382128:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2410 00000100:00080000:1.0:1775765789.382132:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765789.382135:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765789.382139:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765789.382157:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765789/real 1775765789] req@ffff8f994f2e6680 x1862022967157376/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765789.382169:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967157760/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765789.382181:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967157376/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765789.382192:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765789.382197:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765789.382202:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967157376:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765789.382226:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967157504:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765789.382248:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765789.382258:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765789/real 1775765789] req@ffff8f994f2e5f80 x1862022967157504/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765789.382268:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967157504/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765789.382274:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765789.382277:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765789.382280:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967157504:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765789.382284:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967157632:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765789.382297:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765789.382304:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765789/real 1775765789] req@ffff8f994f2e6a00 x1862022967157632/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765789.382328:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967157632/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765789.382338:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765789.382343:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765789.382348:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967157632:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765789.382355:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967157760:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765789.382393:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765789.382409:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765789/real 1775765789] req@ffff8f994f2e4000 x1862022967157760/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765789.382421:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967157760/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765789.382454:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765789.382456:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765789.382459:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967157760:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765790.277924:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765790.278010:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765791.301826:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765791.301923:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765792.325810:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765792.325928:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765793.349874:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765793.349928:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765794.373897:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765794.373994:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765794.501976:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765794.501989:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765794.501996:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765794.502010:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2416 00000100:00080000:3.0:1775765794.502087:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765794.502099:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765794.502149:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967157888/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765794.502198:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765794.502204:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765794.502208:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765794.502220:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2416 00000100:00080000:3.0:1775765794.502230:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765794.502240:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765794.502251:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967157888:192.168.202.146@tcp:250:kworker.0 00000008:00100000:3.0:1775765794.502260:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765794.502301:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967158016/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765794.502320:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765794.502326:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765794.502328:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765794.502335:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2416 00000100:00080000:3.0:1775765794.502340:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765794.502345:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765794.502361:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765794.502375:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967158144/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765794.502390:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765794.502395:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765794.502398:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765794.502403:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2416 00000100:00080000:3.0:1775765794.502407:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765794.502412:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765794.502418:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775765794.502536:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765794.502582:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765794/real 1775765794] req@ffff8f9970edca80 x1862022967157888/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765794.502604:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967157888/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765794.502616:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765794.502621:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765794.502625:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967157888:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775765794.502655:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967158016:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775765794.502679:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765794.502693:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765794/real 1775765794] req@ffff8f9970edea00 x1862022967158016/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765794.502706:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967158016/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765794.502715:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765794.502719:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765794.502723:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967158016:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775765794.502729:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967158144:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765794.502816:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765794.502834:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765794/real 1775765794] req@ffff8f9970edf800 x1862022967158144/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765794.502846:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967158272/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765794.502903:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967158144/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765794.502915:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765794.502924:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765794.502929:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967158144:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775765794.502964:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967158272:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765794.502987:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765794.503002:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765794/real 1775765794] req@ffff8f9970edd880 x1862022967158272/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765794.503037:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967158272/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765794.503047:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765794.503051:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765794.503054:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967158272:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765795.397837:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765795.397926:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765796.421872:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765796.422035:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765797.445864:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765797.445947:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765798.469869:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765798.469946:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765799.493810:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765799.493891:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765799.621842:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765799.621853:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765799.621857:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765799.621865:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2421 00000100:00080000:1.0:1775765799.621875:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765799.621903:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765799.621935:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967158400/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765799.621959:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765799.621963:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765799.621973:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765799.621978:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2421 00000100:00080000:1.0:1775765799.621982:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765799.621987:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765799.621996:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765799.621999:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967158400:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775765799.622008:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967158528/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765799.622018:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765799.622021:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765799.622022:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765799.622026:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2421 00000100:00080000:1.0:1775765799.622029:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765799.622032:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765799.622037:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765799.622045:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967158656/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765799.622053:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765799.622056:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:2.0:1775765799.622060:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775765799.622072:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765799.622076:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2421 00000100:00080000:1.0:1775765799.622079:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765799.622082:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765799.622087:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765799.622096:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967158784/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765799.622097:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765799/real 1775765799] req@ffff8f994f2e5180 x1862022967158400/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765799.622115:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967158400/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765799.622125:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765799.622130:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765799.622134:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967158400:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765799.622157:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967158528:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765799.622175:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765799.622184:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765799/real 1775765799] req@ffff8f994f2e7800 x1862022967158528/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765799.622192:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967158528/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765799.622198:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765799.622200:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765799.622202:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967158528:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765799.622206:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967158656:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765799.622216:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765799.622222:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765799/real 1775765799] req@ffff8f994f2e4a80 x1862022967158656/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765799.622245:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967158656/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765799.622251:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765799.622253:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765799.622255:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967158656:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765799.622258:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967158784:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765799.622269:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765799.622276:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765799/real 1775765799] req@ffff8f994f2e6d80 x1862022967158784/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765799.622283:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967158784/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765799.622288:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765799.622290:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765799.622291:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967158784:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775765800.517938:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765800.517942:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775765801.541830:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765801.541943:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765802.565891:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765802.565998:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765803.589855:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765803.589996:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765804.613906:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765804.614046:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765804.742815:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765804.742833:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765804.742841:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765804.742859:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2426 00000100:00080000:0.0:1775765804.742878:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765804.742889:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765804.743032:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967158912/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765804.743087:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765804.743095:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765804.743099:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765804.743109:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2426 00000100:00080000:0.0:1775765804.743117:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765804.743126:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765804.743141:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765804.743182:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967159040/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765804.743204:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967158912:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775765804.743211:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765804.743225:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765804.743229:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765804.743238:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2426 00000100:00080000:0.0:1775765804.743245:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765804.743253:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765804.743266:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765804.743290:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967159168/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765804.743309:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765804.743315:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765804.743321:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1775765804.743325:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775765804.743367:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2426 00000100:00080000:0.0:1775765804.743377:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765804.743385:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765804.743397:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765804.743397:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765804/real 1775765804] req@ffff8f9970a77100 x1862022967158912/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765804.743427:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967159296/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765804.743487:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967158912/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765804.743511:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765804.743520:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765804.743528:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967158912:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765804.743566:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967159040:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765804.743604:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765804.743622:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765804/real 1775765804] req@ffff8f9970a76300 x1862022967159040/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765804.743649:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967159040/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765804.743660:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765804.743664:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765804.743680:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967159040:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765804.743688:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967159168:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765804.743733:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765804.743815:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765804/real 1775765804] req@ffff8f9970a75180 x1862022967159168/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765804.743842:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967159168/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765804.743898:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765804.743903:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765804.743907:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967159168:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765804.743937:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967159296:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765804.743986:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765804.744002:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765804/real 1775765804] req@ffff8f9970a75500 x1862022967159296/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765804.744017:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967159296/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765804.744028:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765804.744032:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765804.744036:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967159296:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765805.637818:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765805.637965:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765806.661873:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765806.661973:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765807.685902:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765807.685912:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765808.709895:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765808.709963:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765809.733830:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765809.733962:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765809.861882:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765809.861890:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765809.861894:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765809.861905:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2431 00000100:00080000:2.0:1775765809.861915:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765809.861920:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765809.861986:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1c00 x1862022967159424/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765809.862013:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765809.862015:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765809.862017:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765809.862020:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2431 00000100:00080000:2.0:1775765809.862023:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765809.862025:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765809.862032:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765809.862040:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3b80 x1862022967159552/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765809.862048:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765809.862051:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765809.862053:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765809.862055:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2431 00000100:00080000:2.0:1775765809.862057:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765809.862060:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765809.862063:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765809.862071:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1880 x1862022967159680/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765809.862079:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765809.862085:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1775765809.862087:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967159424:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775765809.862087:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765809.862101:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2431 00000100:00080000:2.0:1775765809.862104:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765809.862107:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765809.862111:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765809.862125:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2a00 x1862022967159808/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765809.862174:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765809.862230:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765809/real 1775765809] req@ffff8f99703d1c00 x1862022967159424/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765809.862248:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1c00 x1862022967159424/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765809.862258:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765809.862263:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765809.862266:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967159424:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775765809.862272:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967159552:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775765809.862480:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765809.862510:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765809/real 1775765809] req@ffff8f99703d3b80 x1862022967159552/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765809.862531:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3b80 x1862022967159552/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765809.862542:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765809.862547:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765809.862563:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967159552:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775765809.862589:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967159680:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765809.862607:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765809.862615:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765809/real 1775765809] req@ffff8f99703d1880 x1862022967159680/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765809.862635:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1880 x1862022967159680/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765809.862641:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765809.862643:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765809.862645:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967159680:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775765809.862649:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967159808:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765809.862658:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765809.862664:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765809/real 1775765809] req@ffff8f99703d2a00 x1862022967159808/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765809.862691:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2a00 x1862022967159808/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765809.862697:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765809.862699:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765809.862701:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967159808:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765810.757896:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765810.758019:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765811.781862:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765811.781923:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765812.805876:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765812.805964:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765813.829952:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765813.829997:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765814.854081:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765814.854084:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765814.981832:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765814.981843:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765814.981847:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765814.981856:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2436 00000100:00080000:2.0:1775765814.981865:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765814.981870:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765814.981902:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2a00 x1862022967159936/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765814.981925:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765814.981928:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765814.981929:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765814.981933:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2436 00000100:00080000:2.0:1775765814.981935:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765814.981937:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765814.981964:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765814.981968:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967159936:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765814.981977:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1880 x1862022967160064/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765814.981985:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765814.981988:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765814.981989:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765814.981991:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2436 00000100:00080000:2.0:1775765814.981995:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765814.981997:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765814.982001:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765814.982008:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3b80 x1862022967160192/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765814.982015:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765814.982017:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765814.982018:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765814.982022:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2436 00000800:00000100:1.0:1775765814.982027:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775765814.982028:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765814.982032:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765814.982035:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765814.982044:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1c00 x1862022967160320/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765814.982076:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765814/real 1775765814] req@ffff8f99703d2a00 x1862022967159936/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765814.982092:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2a00 x1862022967159936/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765814.982100:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765814.982104:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765814.982107:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967159936:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775765814.982124:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967160064:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775765814.982138:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765814.982146:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765814/real 1775765814] req@ffff8f99703d1880 x1862022967160064/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765814.982175:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1880 x1862022967160064/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765814.982181:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765814.982183:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765814.982186:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967160064:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775765814.982190:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967160192:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765814.982203:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765814.982209:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765814/real 1775765814] req@ffff8f99703d3b80 x1862022967160192/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765814.982217:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3b80 x1862022967160192/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765814.982221:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765814.982223:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765814.982225:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967160192:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775765814.982229:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967160320:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765814.982238:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765814.982244:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765814/real 1775765814] req@ffff8f99703d1c00 x1862022967160320/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765814.982251:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1c00 x1862022967160320/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765814.982271:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765814.982273:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765814.982275:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967160320:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765815.877841:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765815.877891:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765816.901928:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765816.902054:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765817.925886:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765817.926023:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765818.949886:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765818.950003:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765819.973925:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765819.974043:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765820.101855:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765820.101864:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765820.101869:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765820.101877:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2441 00000100:00080000:1.0:1775765820.101887:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765820.101893:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765820.101935:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967160448/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765820.101960:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765820.101963:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765820.101965:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765820.101968:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2441 00000100:00080000:1.0:1775765820.101981:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765820.101984:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765820.101992:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765820.102002:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967160576/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765820.102003:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967160448:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775765820.102059:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765820.102062:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765820.102063:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765820.102067:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2441 00000100:00080000:1.0:1775765820.102070:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765820.102082:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:2.0:1775765820.102083:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:1.0:1775765820.102087:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765820.102097:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967160704/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765820.102104:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765820.102107:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765820.102108:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765820.102111:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2441 00000100:00080000:1.0:1775765820.102113:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765820.102115:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765820.102119:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765820.102136:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967160832/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765820.102155:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765820/real 1775765820] req@ffff8f994f2e6d80 x1862022967160448/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765820.102178:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967160448/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765820.102191:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765820.102196:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765820.102201:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967160448:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765820.102228:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967160576:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765820.102253:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765820.102266:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765820/real 1775765820] req@ffff8f994f2e4a80 x1862022967160576/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765820.102280:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967160576/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765820.102289:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765820.102293:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765820.102297:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967160576:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765820.102303:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967160704:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765820.102350:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765820.102363:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765820/real 1775765820] req@ffff8f994f2e7800 x1862022967160704/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765820.102377:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967160704/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765820.102385:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765820.102398:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765820.102402:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967160704:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765820.102408:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967160832:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765820.102427:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765820.102438:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765820/real 1775765820] req@ffff8f994f2e5180 x1862022967160832/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765820.102469:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967160832/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765820.102478:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765820.102482:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765820.102485:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967160832:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765820.997842:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765820.997907:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765822.021903:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765822.021933:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765823.045835:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765823.045897:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765824.069850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765824.069915:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765825.093865:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765825.094137:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765825.221907:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765825.221920:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765825.221926:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765825.221939:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2446 00000100:00080000:0.0:1775765825.221954:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765825.221961:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765825.222011:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967160960/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765825.222046:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765825.222051:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765825.222053:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765825.222059:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2446 00000100:00080000:0.0:1775765825.222063:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765825.222067:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765825.222080:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765825.222228:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967160960:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765825.222364:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967161088/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765825.222375:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775765825.222399:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765825.222404:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765825.222408:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765825.222416:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2446 00000100:00080000:0.0:1775765825.222426:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765825.222433:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765825.222443:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765825.222446:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765825/real 1775765825] req@ffff8f9970a77480 x1862022967160960/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765825.222461:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967161216/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765825.222475:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775765825.222479:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967160960/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765825.222484:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765825.222490:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765825.222504:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765825.222510:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765825.222515:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967160960:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775765825.222522:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2446 00000100:00080000:0.0:1775765825.222529:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765825.222533:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765825.222538:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765825.222563:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967161088:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775765825.222567:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967161344/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765825.222599:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765825.222621:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765825/real 1775765825] req@ffff8f9970a77800 x1862022967161088/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765825.222650:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967161088/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765825.222665:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765825.222670:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765825.222676:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967161088:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765825.222687:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967161216:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765825.222807:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765825.222825:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765825/real 1775765825] req@ffff8f9970a76a00 x1862022967161216/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765825.222867:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967161216/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765825.222876:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765825.222880:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765825.222883:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967161216:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765825.222900:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967161344:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765825.222923:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765825.222933:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765825/real 1775765825] req@ffff8f9970a74380 x1862022967161344/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765825.222944:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967161344/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765825.222952:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765825.222955:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765825.222958:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967161344:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765826.117811:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765826.117886:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765827.141842:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765827.141910:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765828.165841:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765828.165890:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765829.189911:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765829.189950:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775765830.213843:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765830.213899:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765830.341797:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765830.341806:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765830.341845:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765830.341856:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2451 00000100:00080000:0.0:1775765830.341867:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765830.341874:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765830.341910:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967161472/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765830.341938:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765830.341941:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765830.341943:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765830.341947:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2451 00000100:00080000:0.0:1775765830.341950:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765830.341955:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765830.341964:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765830.341976:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967161600/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765830.341986:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765830.341989:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765830.341991:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765830.341994:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2451 00000100:00080000:0.0:1775765830.341998:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775765830.342002:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967161472:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775765830.342003:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765830.342011:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765830.342020:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967161728/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765830.342046:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765830.342050:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765830.342052:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765830.342058:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2451 00000100:00080000:0.0:1775765830.342062:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1775765830.342063:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775765830.342071:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765830.342078:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765830.342094:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967161856/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765830.342103:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765830/real 1775765830] req@ffff8f9970a75c00 x1862022967161472/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765830.342131:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967161472/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765830.342139:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765830.342143:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765830.342156:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967161472:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775765830.342162:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967161600:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775765830.342177:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765830.342195:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765830/real 1775765830] req@ffff8f9970a74700 x1862022967161600/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765830.342203:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967161600/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765830.342209:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765830.342211:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765830.342213:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967161600:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775765830.342231:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967161728:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765830.342275:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765830.342284:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765830/real 1775765830] req@ffff8f9970a76d80 x1862022967161728/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765830.342293:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967161728/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765830.342300:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765830.342304:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765830.342306:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967161728:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775765830.342311:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967161856:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765830.342322:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765830.342335:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765830/real 1775765830] req@ffff8f9970a74380 x1862022967161856/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765830.342344:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967161856/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765830.342350:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765830.342352:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765830.342355:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967161856:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765831.237868:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765831.237924:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765832.261808:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765832.261870:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765833.285861:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765833.285917:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765834.309817:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765834.309854:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765835.333867:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765835.333958:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765835.461904:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765835.461916:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765835.461923:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765835.461935:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2457 00000100:00080000:2.0:1775765835.461950:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765835.461958:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765835.462003:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2300 x1862022967161984/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765835.462040:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765835.462045:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765835.462047:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765835.462052:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2457 00000100:00080000:2.0:1775765835.462056:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765835.462060:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765835.462072:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765835.462086:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3480 x1862022967162112/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765835.462101:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765835.462104:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765835.462107:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765835.462114:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2457 00000100:00100000:1.0:1775765835.462116:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967161984:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775765835.462117:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765835.462127:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765835.462197:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765835.462220:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3100 x1862022967162240/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765835.462234:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765835.462239:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765835.462242:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765835.462247:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2457 00000100:00080000:2.0:1775765835.462253:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765835.462257:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765835.462264:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775765835.462276:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765835.462283:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1180 x1862022967162368/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765835.462352:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765835/real 1775765835] req@ffff8f99703d2300 x1862022967161984/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765835.462377:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2300 x1862022967161984/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765835.462391:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765835.462396:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765835.462401:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967161984:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775765835.462430:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967162112:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775765835.462458:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765835.462472:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765835/real 1775765835] req@ffff8f99703d3480 x1862022967162112/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765835.462488:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3480 x1862022967162112/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765835.462498:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765835.462501:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765835.462505:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967162112:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775765835.462513:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967162240:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765835.462532:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765835.462543:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765835/real 1775765835] req@ffff8f99703d3100 x1862022967162240/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765835.462592:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3100 x1862022967162240/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765835.462602:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765835.462606:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765835.462610:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967162240:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775765835.462662:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967162368:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775765835.462702:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775765835.462742:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765835/real 1775765835] req@ffff8f99703d1180 x1862022967162368/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765835.462768:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1180 x1862022967162368/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765835.462838:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765835.462845:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775765835.462851:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967162368:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765836.357866:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765836.357900:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765837.381828:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765837.381873:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765838.405808:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765838.405893:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765839.429851:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765839.429926:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765840.453818:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765840.453904:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765840.581879:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765840.581889:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765840.581894:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765840.581904:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2462 00000100:00080000:1.0:1775765840.581949:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765840.581956:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765840.581994:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967162496/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765840.582030:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765840.582034:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765840.582036:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765840.582041:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2462 00000100:00080000:1.0:1775765840.582045:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765840.582049:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765840.582059:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765840.582073:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967162624/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765840.582084:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765840.582088:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765840.582090:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765840.582095:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2462 00000100:00080000:1.0:1775765840.582099:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775765840.582102:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967162496:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775765840.582106:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765840.582115:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765840.582131:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967162752/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765840.582143:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1775765840.582164:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775765840.582167:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765840.582170:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765840.582176:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2462 00000100:00080000:1.0:1775765840.582184:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765840.582189:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765840.582194:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765840.582207:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967162880/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765840.582213:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765840/real 1775765840] req@ffff8f994f2e4000 x1862022967162496/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765840.582247:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967162496/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765840.582255:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765840.582276:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765840.582279:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967162496:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765840.582285:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967162624:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765840.582301:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765840.582309:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765840/real 1775765840] req@ffff8f994f2e6a00 x1862022967162624/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765840.582317:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967162624/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765840.582322:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765840.582324:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765840.582327:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967162624:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765840.582344:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967162752:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765840.582356:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765840.582365:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765840/real 1775765840] req@ffff8f994f2e5f80 x1862022967162752/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765840.582378:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967162752/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765840.582386:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765840.582389:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765840.582392:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967162752:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765840.582398:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967162880:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765840.582434:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765840.582445:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765840/real 1775765840] req@ffff8f994f2e6680 x1862022967162880/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765840.582453:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967162880/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765840.582458:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765840.582460:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765840.582462:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967162880:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765841.477834:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765841.477935:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765842.501822:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765842.501904:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765843.525935:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765843.525973:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765844.549879:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765844.549917:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765845.573918:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765845.574027:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765845.701910:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765845.701922:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765845.701929:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765845.701942:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2467 00000100:00080000:0.0:1775765845.701967:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765845.701976:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765845.702024:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967163008/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765845.702062:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765845.702068:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765845.702113:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765845.702121:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2467 00000100:00080000:0.0:1775765845.702126:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775765845.702132:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967163008:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775765845.702137:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765845.702157:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765845.702187:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967163136/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765845.702224:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765845.702231:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765845.702235:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765845.702243:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2467 00000100:00080000:0.0:1775765845.702252:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765845.702261:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765845.702275:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765845.702296:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967163136:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775765845.702307:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967163264/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765845.702352:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765845.702360:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1775765845.702364:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967163264:192.168.202.146@tcp:8:kworker.0 00000100:00080000:0.0:1775765845.702367:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765845.702383:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2467 00000100:00080000:0.0:1775765845.702395:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765845.702405:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765845.702417:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765845.702471:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967163392/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765845.702580:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967163392:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775765846.597914:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765846.597982:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:3.0:1775765847.621882:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765847.621905:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765848.645838:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765848.645927:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1775765848.774121:0:2366:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:0.0:1775765848.774169:0:2366:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1775765848.774188:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775765848.774244:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775765848.774255:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775765848.774271:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:2.0:1775765848.774338:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765845/real 1775765848] req@ffff8f9970a75500 x1862022967163392/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765848.774369:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967163392/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765848.774383:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765848.774430:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765848.774435:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967163392:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765848.774444:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765845/real 1775765848] req@ffff8f9970a77480 x1862022967163264/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765848.774459:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967163264/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765848.774469:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765848.774473:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765848.774477:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967163264:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765848.774484:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765845/real 1775765848] req@ffff8f9970a77800 x1862022967163136/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765848.774516:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967163136/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765848.774526:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765848.774531:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765848.774535:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967163136:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765848.774543:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765845/real 1775765848] req@ffff8f9970a76a00 x1862022967163008/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765848.774560:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967163008/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765848.774567:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765848.774589:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765848.774594:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967163008:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765849.669917:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765849.669966:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765850.693918:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765850.693946:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00080000:3.0:1775765850.821983:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765850.821997:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765850.822003:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765850.822017:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2472 00000100:00080000:3.0:1775765850.822031:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765850.822041:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765850.822084:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967163520/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765850.822125:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765850.822131:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765850.822134:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765850.822140:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2472 00000100:00080000:3.0:1775765850.822146:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765850.822152:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765850.822165:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765850.822184:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967163648/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765850.822198:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967163520:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775765850.822204:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765850.822213:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765850.822216:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765850.822222:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2472 00000100:00080000:3.0:1775765850.822229:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765850.822234:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765850.822243:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765850.822277:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967163776/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765850.822292:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775765850.822297:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765850.822303:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765850.822306:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765850.822314:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2472 00000100:00080000:3.0:1775765850.822320:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765850.822325:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765850.822333:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765850.822351:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967163904/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765850.822381:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765850/real 1775765850] req@ffff8f9970edf480 x1862022967163520/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765850.822411:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967163520/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765850.822427:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765850.822434:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765850.822439:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967163520:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765850.822473:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967163648:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765850.822554:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765850.822574:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765850/real 1775765850] req@ffff8f9970ede300 x1862022967163648/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765850.822592:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967163648/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765850.822604:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765850.822609:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765850.822614:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967163648:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765850.822695:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967163776:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765850.822732:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765850.822783:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765850/real 1775765850] req@ffff8f9970edc000 x1862022967163776/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765850.822799:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967163776/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765850.822810:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765850.822814:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765850.822818:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967163776:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765850.822841:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967163904:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765850.822873:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765850.822890:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765850/real 1775765850] req@ffff8f9970ede680 x1862022967163904/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765850.822909:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967163904/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765850.822920:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765850.822925:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765850.822930:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967163904:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775765851.717876:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765851.718019:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765852.741873:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765852.741897:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765853.765828:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765853.765954:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765854.789854:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765854.789970:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765855.813814:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765855.813870:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765855.941866:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765855.941874:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765855.941878:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765855.941886:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2477 00000100:00080000:0.0:1775765855.941896:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765855.941901:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765855.941940:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967164032/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765855.941963:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765855.941966:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765855.941967:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765855.941971:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2477 00000100:00080000:0.0:1775765855.941974:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765855.941977:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765855.941985:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765855.941994:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967164032:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765855.942057:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967164160/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765855.942067:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1775765855.942071:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775765855.942072:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765855.942075:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765855.942078:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2477 00000100:00080000:0.0:1775765855.942083:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765855.942086:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765855.942091:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765855.942099:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967164288/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765855.942111:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765855/real 1775765855] req@ffff8f9970a75180 x1862022967164032/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765855.942117:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765855.942123:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765855.942126:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765855.942132:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2477 00000100:00080000:0.0:1775765855.942136:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765855.942141:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765855.942141:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967164032/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1775765855.942150:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1775765855.942154:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765855.942159:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765855.942162:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967164032:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765855.942169:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967164416/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765855.942185:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967164160:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765855.942203:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765855.942221:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765855/real 1775765855] req@ffff8f9970a76300 x1862022967164160/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765855.942230:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967164160/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765855.942236:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765855.942238:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765855.942240:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967164160:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765855.942244:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967164288:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765855.942256:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765855.942262:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765855/real 1775765855] req@ffff8f9970a77100 x1862022967164288/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765855.942309:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967164288/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765855.942315:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765855.942317:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765855.942319:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967164288:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765855.942323:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967164416:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765855.942337:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765855.942367:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765855/real 1775765855] req@ffff8f9970a76a00 x1862022967164416/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765855.942375:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967164416/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765855.942381:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765855.942383:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765855.942385:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967164416:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765856.837840:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765856.837920:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765857.861866:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765857.861875:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775765858.885891:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765858.886026:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765859.909812:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765859.909943:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765860.933903:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765860.934361:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765861.061842:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765861.061850:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765861.061911:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765861.061920:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2482 00000100:00080000:0.0:1775765861.061931:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765861.061937:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765861.061970:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967164544/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765861.061991:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765861.061994:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765861.061995:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765861.062009:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2482 00000100:00080000:0.0:1775765861.062012:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775765861.062059:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967164544:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775765861.062080:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765861.062089:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765861.062099:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967164672/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765861.062106:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765861.062109:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765861.062110:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765861.062113:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2482 00000100:00080000:0.0:1775765861.062115:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765861.062117:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765861.062120:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765861.062126:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967164800/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765861.062171:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765861.062173:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765861.062175:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765861.062177:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2482 00000100:00080000:0.0:1775765861.062180:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765861.062182:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765861.062185:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765861.062203:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967164928/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765861.062292:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765861.062340:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765861/real 1775765861] req@ffff8f9970a77800 x1862022967164544/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765861.062367:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967164544/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765861.062376:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765861.062391:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765861.062395:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967164544:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765861.062419:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967164672:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765861.062441:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765861.062450:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765861/real 1775765861] req@ffff8f9970a77480 x1862022967164672/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765861.062458:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967164672/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765861.062464:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765861.062466:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765861.062468:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967164672:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765861.062472:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967164800:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765861.062497:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765861.062504:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765861/real 1775765861] req@ffff8f9970a75500 x1862022967164800/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765861.062511:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967164800/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765861.062530:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765861.062533:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765861.062535:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967164800:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765861.062539:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967164928:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765861.062551:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765861.062557:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765861/real 1775765861] req@ffff8f9970a74380 x1862022967164928/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765861.062565:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967164928/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765861.062570:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765861.062572:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765861.062574:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967164928:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765861.957817:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765861.957872:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765862.981882:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765862.981917:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775765864.005941:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765864.006045:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765865.029917:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765865.029974:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765866.053831:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765866.053879:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765866.181833:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765866.181843:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765866.181848:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765866.181857:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2487 00000100:00080000:3.0:1775765866.181867:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765866.181873:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765866.181907:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967165056/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765866.181936:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765866.181940:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765866.181942:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765866.181946:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2487 00000100:00080000:3.0:1775765866.181950:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765866.181954:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765866.181964:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765866.181975:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967165056:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765866.181980:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967165184/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765866.181992:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765866.181995:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765866.181996:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765866.181999:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2487 00000100:00080000:3.0:1775765866.182002:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765866.182005:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765866.182009:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765866.182024:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967165312/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765866.182036:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765866.182040:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765866.182053:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765866.182059:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2487 00000100:00080000:3.0:1775765866.182065:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765866.182070:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765866.182076:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775765866.182085:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765866.182092:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967165440/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765866.182123:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765866/real 1775765866] req@ffff8f9970edc700 x1862022967165056/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765866.182140:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967165056/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765866.182149:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765866.182152:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765866.182156:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967165056:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765866.182192:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967165184:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775765866.182216:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765866.182265:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765866/real 1775765866] req@ffff8f9970eded80 x1862022967165184/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765866.182276:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967165184/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765866.182284:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765866.182286:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765866.182299:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967165184:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775765866.182305:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967165312:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765866.182322:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765866.182349:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765866/real 1775765866] req@ffff8f9970edce00 x1862022967165312/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765866.182388:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967165312/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765866.182395:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765866.182398:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765866.182401:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967165312:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775765866.182407:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967165440:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775765866.182422:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775765866.182429:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765866/real 1775765866] req@ffff8f9970edd500 x1862022967165440/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765866.182437:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967165440/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765866.182456:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765866.182458:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775765866.182460:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967165440:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765867.077859:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765867.077981:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765868.101835:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765868.101881:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765869.125824:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765869.125859:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765870.149902:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765870.150062:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765871.173816:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765871.173867:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765871.302228:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765871.302240:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765871.302245:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765871.302257:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2492 00000100:00080000:0.0:1775765871.302268:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765871.302321:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765871.302388:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967165568/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765871.302428:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765871.302433:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765871.302435:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765871.302440:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2492 00000100:00080000:0.0:1775765871.302445:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765871.302449:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765871.302457:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765871.302481:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967165696/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765871.302489:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765871.302504:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765871.302505:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765871.302509:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2492 00000100:00080000:0.0:1775765871.302514:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765871.302518:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765871.302524:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765871.302548:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967165824/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765871.302557:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765871.302560:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765871.302562:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765871.302565:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2492 00000100:00080000:0.0:1775765871.302568:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765871.302572:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765871.302576:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765871.302588:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967165952/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765871.302680:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967165568:192.168.202.146@tcp:250:kworker.0 00000800:00000100:2.0:1775765871.302825:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765871.302876:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765871/real 1775765871] req@ffff8f9970a76d80 x1862022967165568/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765871.302895:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967165568/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765871.302905:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765871.302909:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765871.302913:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967165568:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765871.302921:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967165696:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765871.302945:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765871.302959:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765871/real 1775765871] req@ffff8f9970a74700 x1862022967165696/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765871.302973:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967165696/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765871.302983:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765871.302986:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765871.302990:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967165696:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765871.302996:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967165824:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765871.303028:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765871.303037:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765871/real 1775765871] req@ffff8f9970a75c00 x1862022967165824/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765871.303060:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967165824/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765871.303065:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765871.303067:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765871.303069:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967165824:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765871.303074:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967165952:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765871.303086:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765871.303092:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765871/real 1775765871] req@ffff8f9970a74380 x1862022967165952/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765871.303099:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967165952/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765871.303104:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765871.303106:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765871.303108:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967165952:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765872.197867:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765872.197889:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765873.221855:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765873.222010:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765874.245962:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765874.246798:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775765875.269819:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765875.269875:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765876.293882:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765876.293968:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765876.421865:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765876.421881:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765876.421886:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765876.421897:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2498 00000100:00080000:3.0:1775765876.421910:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765876.421918:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765876.422048:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf100 x1862022967166080/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765876.422074:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765876.422078:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765876.422079:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765876.422083:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2498 00000100:00080000:3.0:1775765876.422087:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765876.422091:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765876.422101:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765876.422121:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967166080:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765876.422162:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddc00 x1862022967166208/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765876.422176:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765876.422180:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765876.422182:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765876.422186:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2498 00000100:00080000:3.0:1775765876.422191:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775765876.422198:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775765876.422202:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765876.422209:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765876.422225:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967166336/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765876.422249:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765876.422254:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765876.422257:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765876.422262:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2498 00000100:00080000:3.0:1775765876.422267:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765876.422271:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765876.422278:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765876.422282:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765876/real 1775765876] req@ffff8f9970edf100 x1862022967166080/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765876.422313:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967166464/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765876.422368:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf100 x1862022967166080/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765876.422380:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765876.422386:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765876.422391:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967166080:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765876.422434:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967166208:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765876.422456:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765876.422466:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765876/real 1775765876] req@ffff8f9970eddc00 x1862022967166208/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765876.422474:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddc00 x1862022967166208/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765876.422480:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765876.422483:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765876.422485:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967166208:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765876.422490:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967166336:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765876.422501:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765876.422507:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765876/real 1775765876] req@ffff8f9970edd500 x1862022967166336/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765876.422514:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967166336/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765876.422536:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765876.422538:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765876.422540:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967166336:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765876.422544:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967166464:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765876.422563:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765876.422570:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765876/real 1775765876] req@ffff8f9970edce00 x1862022967166464/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765876.422578:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967166464/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765876.422583:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765876.422586:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765876.422588:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967166464:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765877.317856:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765877.317926:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765878.341843:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765878.341911:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765879.365876:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765879.365885:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765880.389836:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765880.389942:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765881.413836:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765881.413912:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765881.541893:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765881.541906:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765881.541912:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765881.541925:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2503 00000100:00080000:0.0:1775765881.541940:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765881.542110:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765881.542153:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967166592/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765881.542213:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765881.542219:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765881.542221:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765881.542228:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2503 00000100:00100000:2.0:1775765881.542232:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967166592:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775765881.542240:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765881.542247:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765881.542258:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765881.542276:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967166720/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765881.542289:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765881.542292:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765881.542295:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765881.542299:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2503 00000800:00000100:2.0:1775765881.542304:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775765881.542305:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765881.542313:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765881.542318:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765881.542329:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967166848/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765881.542336:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765881.542339:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765881.542357:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765881.542362:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2503 00000100:00100000:2.0:1775765881.542362:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765881/real 1775765881] req@ffff8f9970a75500 x1862022967166592/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765881.542367:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765881.542372:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765881.542379:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765881.542390:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967166976/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765881.542397:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967166592/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765881.542425:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765881.542429:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765881.542432:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967166592:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765881.542452:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967166720:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765881.542471:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765881.542480:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765881/real 1775765881] req@ffff8f9970a77480 x1862022967166720/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765881.542497:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967166720/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765881.542503:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765881.542505:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765881.542507:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967166720:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765881.542512:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967166848:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765881.542525:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765881.542531:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765881/real 1775765881] req@ffff8f9970a77800 x1862022967166848/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765881.542538:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967166848/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765881.542543:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765881.542545:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765881.542547:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967166848:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765881.542563:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967166976:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765881.542575:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765881.542582:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765881/real 1775765881] req@ffff8f9970a76a00 x1862022967166976/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765881.542589:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967166976/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765881.542595:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765881.542597:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765881.542599:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967166976:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765882.437891:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765882.437951:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765883.461839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765883.461923:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765884.485883:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765884.485966:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765885.509878:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765885.509949:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765886.533912:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765886.533940:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00080000:3.0:1775765886.661872:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765886.661880:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765886.661885:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765886.661894:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2508 00000100:00080000:3.0:1775765886.661904:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765886.661909:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765886.661966:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967167104/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765886.661990:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765886.661993:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765886.661994:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765886.661997:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2508 00000100:00080000:3.0:1775765886.662002:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765886.662005:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765886.662013:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765886.662024:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967167232/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765886.662033:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765886.662036:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1775765886.662038:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967167104:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775765886.662038:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765886.662045:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2508 00000100:00080000:3.0:1775765886.662055:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765886.662061:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765886.662065:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765886.662074:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967167360/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765886.662083:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765886.662086:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765886.662087:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765886.662091:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2508 00000100:00080000:3.0:1775765886.662093:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765886.662097:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765886.662101:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765886.662109:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967167488/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765886.662154:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765886.662215:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765886/real 1775765886] req@ffff8f9970eded80 x1862022967167104/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765886.662235:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967167104/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765886.662248:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765886.662253:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765886.662259:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967167104:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765886.662386:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967167232:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765886.662441:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765886.662460:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765886/real 1775765886] req@ffff8f9970edc700 x1862022967167232/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765886.662472:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967167232/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765886.662479:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765886.662482:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765886.662485:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967167232:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765886.662489:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967167360:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765886.662502:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765886.662509:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765886/real 1775765886] req@ffff8f9970ede680 x1862022967167360/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765886.662516:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967167360/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765886.662521:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765886.662523:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765886.662525:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967167360:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765886.662528:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967167488:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765886.662539:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765886.662544:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765886/real 1775765886] req@ffff8f9970edc000 x1862022967167488/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765886.662568:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967167488/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765886.662574:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765886.662576:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765886.662578:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967167488:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765887.557817:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765887.557885:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765888.581848:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765888.581905:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765889.605809:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765889.605902:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765890.629850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765890.629898:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765891.653801:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765891.653875:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765891.781838:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765891.781849:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765891.781854:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765891.781864:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2513 00000100:00080000:3.0:1775765891.781874:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765891.781880:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765891.781914:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967167616/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765891.781939:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765891.781941:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765891.781943:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765891.781946:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2513 00000100:00080000:3.0:1775765891.781949:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765891.781951:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765891.781958:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765891.781968:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967167744/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765891.781976:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765891.781979:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765891.781980:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765891.781985:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967167616:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775765891.781985:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2513 00000100:00080000:3.0:1775765891.781988:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765891.781994:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765891.781999:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765891.782009:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967167872/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765891.782019:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765891.782023:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765891.782025:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765891.782029:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2513 00000100:00080000:3.0:1775765891.782044:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765891.782049:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765891.782054:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765891.782059:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765891.782072:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967168000/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765891.782125:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765891/real 1775765891] req@ffff8f9970ede300 x1862022967167616/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765891.782146:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967167616/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765891.782157:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765891.782161:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765891.782165:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967167616:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765891.782190:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967167744:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765891.782227:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765891.782242:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765891/real 1775765891] req@ffff8f9970edf480 x1862022967167744/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765891.782292:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967167744/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765891.782301:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765891.782304:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765891.782307:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967167744:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765891.782313:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967167872:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765891.782336:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765891.782347:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765891/real 1775765891] req@ffff8f9970edd880 x1862022967167872/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765891.782359:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967167872/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765891.782368:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765891.782371:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765891.782374:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967167872:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765891.782380:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967168000:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765891.782399:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765891.782410:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765891/real 1775765891] req@ffff8f9970edf800 x1862022967168000/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765891.782421:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967168000/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765891.782450:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765891.782454:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765891.782457:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967168000:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765892.677839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765892.677898:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765893.701913:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765893.701922:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765894.725857:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765894.725925:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765895.749813:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765895.749894:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765896.773830:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765896.773994:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765896.901879:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765896.901887:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765896.901891:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765896.901899:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2518 00000100:00080000:1.0:1775765896.901909:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765896.901914:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765896.901995:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967168128/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765896.902026:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765896.902028:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765896.902030:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765896.902050:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2518 00000100:00080000:1.0:1775765896.902054:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765896.902057:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765896.902066:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765896.902089:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967168128:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775765896.902105:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967168256/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765896.902113:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765896.902116:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765896.902117:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765896.902119:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2518 00000100:00080000:1.0:1775765896.902122:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765896.902124:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765896.902128:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765896.902142:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967168384/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765896.902149:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765896.902151:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765896.902152:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765896.902154:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2518 00000100:00080000:1.0:1775765896.902161:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765896.902164:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765896.902167:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765896.902181:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967168512/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765896.902223:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765896.902282:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765896/real 1775765896] req@ffff8f994f2e4700 x1862022967168128/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765896.902299:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967168128/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765896.902308:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765896.902312:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765896.902317:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967168128:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765896.902337:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967168256:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765896.902356:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765896.902368:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765896/real 1775765896] req@ffff8f994f2e6300 x1862022967168256/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765896.902381:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967168256/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765896.902407:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765896.902411:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765896.902415:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967168256:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765896.902422:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967168384:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765896.902443:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765896.902451:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765896/real 1775765896] req@ffff8f994f2e4e00 x1862022967168384/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765896.902459:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967168384/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765896.902465:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765896.902467:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765896.902469:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967168384:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765896.902472:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967168512:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765896.902497:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765896.902504:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765896/real 1775765896] req@ffff8f994f2e7b80 x1862022967168512/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765896.902511:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967168512/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765896.902516:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765896.902518:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765896.902529:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967168512:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765897.797809:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765897.797860:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765898.821866:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765898.821968:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765899.845853:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765899.845910:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765900.869894:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765900.869933:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765901.893864:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765901.893938:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765902.021814:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765902.021823:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765902.021827:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765902.021836:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2523 00000100:00080000:1.0:1775765902.021846:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765902.021852:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765902.021887:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967168640/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765902.021913:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765902.021916:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765902.021917:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765902.021920:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2523 00000100:00080000:1.0:1775765902.021922:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765902.021926:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765902.021934:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765902.021944:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967168768/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765902.021963:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967168640:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775765902.022006:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765902.022009:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765902.022010:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765902.022013:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2523 00000100:00080000:1.0:1775765902.022017:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765902.022019:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765902.022024:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765902.022033:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967168896/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765902.022039:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765902.022041:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765902.022042:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765902.022044:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2523 00000100:00080000:1.0:1775765902.022046:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765902.022049:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765902.022051:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765902.022057:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967169024/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765902.022069:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765902.022145:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765902/real 1775765902] req@ffff8f994f2e7b80 x1862022967168640/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765902.022169:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967168640/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765902.022184:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765902.022188:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765902.022192:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967168640:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765902.022219:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967168768:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765902.022247:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765902.022257:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765902/real 1775765902] req@ffff8f994f2e4e00 x1862022967168768/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765902.022277:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967168768/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765902.022283:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765902.022286:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765902.022351:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967168768:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765902.022358:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967168896:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765902.022377:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765902.022394:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765902/real 1775765902] req@ffff8f994f2e6300 x1862022967168896/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765902.022403:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967168896/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765902.022410:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765902.022413:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765902.022415:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967168896:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765902.022436:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967169024:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765902.022477:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765902.022487:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765902/real 1775765902] req@ffff8f994f2e4700 x1862022967169024/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765902.022496:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967169024/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765902.022503:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765902.022506:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765902.022509:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967169024:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765902.917820:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765902.917903:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765903.941826:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765903.941897:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765904.965841:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765904.965932:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765905.989902:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765905.990066:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765907.013891:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765907.014006:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765907.141843:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765907.141853:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765907.141859:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765907.141871:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2528 00000100:00080000:3.0:1775765907.141887:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765907.141970:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765907.142035:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967169152/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765907.142060:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765907.142063:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765907.142065:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765907.142069:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2528 00000100:00080000:3.0:1775765907.142073:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765907.142077:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765907.142087:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765907.142106:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967169280/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765907.142113:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967169152:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775765907.142123:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765907.142129:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765907.142130:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765907.142135:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2528 00000100:00080000:3.0:1775765907.142137:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765907.142139:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765907.142144:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765907.142155:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967169408/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765907.142162:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765907.142166:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765907.142168:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765907.142170:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2528 00000100:00080000:3.0:1775765907.142172:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765907.142175:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765907.142178:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765907.142184:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967169536/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765907.142211:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765907.142275:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765907/real 1775765907] req@ffff8f9970edea00 x1862022967169152/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765907.142302:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967169152/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765907.142313:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765907.142317:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765907.142321:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967169152:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765907.142346:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967169280:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765907.142369:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765907.142381:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765907/real 1775765907] req@ffff8f9970edca80 x1862022967169280/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765907.142396:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967169280/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765907.142407:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765907.142412:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765907.142416:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967169280:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765907.142425:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967169408:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765907.142496:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765907.142516:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765907/real 1775765907] req@ffff8f9970edc380 x1862022967169408/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765907.142526:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967169408/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765907.142533:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765907.142536:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765907.142539:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967169408:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765907.142544:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967169536:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765907.142558:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765907.142566:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765907/real 1775765907] req@ffff8f9970edfb80 x1862022967169536/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765907.142575:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967169536/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765907.142581:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765907.142583:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765907.142585:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967169536:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765908.037880:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765908.037891:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765909.061846:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765909.061882:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765910.085838:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765910.086079:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765911.109789:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765911.110001:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765912.133859:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765912.133956:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765912.261965:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765912.261973:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765912.261977:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765912.261985:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2533 00000100:00080000:1.0:1775765912.261994:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765912.261999:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765912.262033:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967169664/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765912.262056:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765912.262058:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765912.262060:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765912.262062:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2533 00000100:00080000:1.0:1775765912.262065:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765912.262068:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765912.262076:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765912.262086:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967169664:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775765912.262104:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967169792/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765912.262116:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765912.262118:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765912.262119:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765912.262121:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2533 00000100:00080000:1.0:1775765912.262124:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765912.262135:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765912.262139:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765912.262146:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967169920/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765912.262152:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765912.262155:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765912.262156:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765912.262159:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2533 00000100:00080000:1.0:1775765912.262163:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765912.262166:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765912.262171:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765912.262183:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967170048/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765912.262208:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967169792:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765912.262275:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967169920:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765912.262308:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967170048:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765913.157812:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765913.157927:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765914.181846:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765914.181962:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765915.205989:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765915.206014:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775765915.334085:0:2368:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:3.0:1775765915.334123:0:2368:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775765915.334159:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:3.0:1775765915.334230:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:3.0:1775765915.334259:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:3.0:1775765915.334271:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:2.0:1775765915.334272:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765912/real 1775765915] req@ffff8f994f2e6680 x1862022967169664/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765915.334341:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967169664/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765915.334355:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765915.334360:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765915.334364:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967169664:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765915.334392:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765912/real 1775765915] req@ffff8f994f2e5f80 x1862022967169792/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765915.334407:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967169792/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765915.334414:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765915.334417:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765915.334421:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967169792:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765915.334428:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765912/real 1775765915] req@ffff8f994f2e6a00 x1862022967169920/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765915.334461:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967169920/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765915.334470:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765915.334474:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765915.334478:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967169920:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765915.334485:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765912/real 1775765915] req@ffff8f994f2e4000 x1862022967170048/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765915.334498:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967170048/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765915.334523:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765915.334527:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765915.334531:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967170048:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765916.229887:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765916.229962:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765917.253948:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765917.254026:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765917.381918:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765917.381932:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765917.381940:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765917.381956:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2538 00000100:00080000:1.0:1775765917.381983:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765917.381993:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765917.382299:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967170176/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765917.382391:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765917.382397:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765917.382401:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765917.382408:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2538 00000100:00080000:1.0:1775765917.382422:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765917.382432:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765917.382450:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765917.382499:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967170304/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765917.382499:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967170176:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775765917.382536:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765917.382541:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765917.382544:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765917.382554:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2538 00000100:00080000:1.0:1775765917.382561:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765917.382567:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765917.382576:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765917.382601:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967170432/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765917.382617:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765917.382624:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765917.382627:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765917.382632:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2538 00000800:00000100:2.0:1775765917.382650:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775765917.382664:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765917.382674:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765917.382684:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765917.382706:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967170560/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765917.382791:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765917/real 1775765917] req@ffff8f994f2e5180 x1862022967170176/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765917.382836:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967170176/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765917.382894:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765917.383043:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765917.383054:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967170176:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765917.383101:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967170304:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765917.383183:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765917.383218:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765917/real 1775765917] req@ffff8f994f2e7800 x1862022967170304/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765917.383242:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967170304/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765917.383257:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765917.383262:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765917.383267:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967170304:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765917.383289:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967170432:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765917.383321:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765917.383338:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765917/real 1775765917] req@ffff8f994f2e4a80 x1862022967170432/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765917.383355:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967170432/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765917.383364:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765917.383369:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765917.383372:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967170432:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765917.383380:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967170560:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765917.383434:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765917.383449:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765917/real 1775765917] req@ffff8f994f2e6d80 x1862022967170560/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765917.383465:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967170560/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765917.383474:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765917.383478:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765917.383482:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967170560:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765918.277908:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765918.277989:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765919.301876:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765919.301949:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765920.325805:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765920.325921:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765921.349888:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765921.349978:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765922.373942:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765922.373983:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765922.501936:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765922.501944:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765922.501948:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765922.501957:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2544 00000100:00080000:3.0:1775765922.501967:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765922.501972:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765922.502091:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd180 x1862022967170688/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765922.502121:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765922.502125:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765922.502127:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765922.502131:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2544 00000100:00080000:3.0:1775765922.502134:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765922.502138:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765922.502147:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765922.502176:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddf80 x1862022967170816/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765922.502194:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967170688:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775765922.502207:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765922.502210:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765922.502212:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765922.502217:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2544 00000100:00080000:3.0:1775765922.502221:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765922.502225:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765922.502231:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765922.502270:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765922.502290:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967170944/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765922.502301:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765922.502305:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765922.502307:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765922.502310:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2544 00000100:00080000:3.0:1775765922.502315:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775765922.502321:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765922/real 1775765922] req@ffff8f9970edd180 x1862022967170688/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765922.502321:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765922.502329:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765922.502350:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd180 x1862022967170688/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765922.502361:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765922.502365:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765922.502368:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967170688:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765922.502385:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967171072/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765922.502389:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967170816:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765922.502429:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765922.502439:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765922/real 1775765922] req@ffff8f9970eddf80 x1862022967170816/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765922.502448:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddf80 x1862022967170816/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765922.502453:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765922.502455:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765922.502457:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967170816:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765922.502461:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967170944:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765922.502472:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765922.502478:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765922/real 1775765922] req@ffff8f9970edfb80 x1862022967170944/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765922.502485:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967170944/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765922.502490:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765922.502491:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765922.502493:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967170944:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765922.502503:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967171072:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765922.502513:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765922.502518:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765922/real 1775765922] req@ffff8f9970edc380 x1862022967171072/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765922.502542:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967171072/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765922.502548:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765922.502550:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765922.502551:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967171072:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765923.397825:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765923.397888:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765924.421888:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765924.421981:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765925.445876:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765925.446027:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765926.469849:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765926.470035:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765927.493849:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765927.493905:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765927.621872:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765927.621882:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765927.621887:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765927.621897:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2549 00000100:00080000:3.0:1775765927.621922:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765927.621932:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765927.621979:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967171200/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765927.622013:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765927.622017:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765927.622019:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765927.622023:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2549 00000100:00080000:3.0:1775765927.622027:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765927.622031:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765927.622041:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765927.622054:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967171328/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765927.622065:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775765927.622069:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967171200:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775765927.622069:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765927.622074:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765927.622082:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2549 00000100:00080000:3.0:1775765927.622087:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765927.622092:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765927.622098:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765927.622117:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967171456/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765927.622131:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765927.622137:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:2.0:1775765927.622142:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775765927.622143:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765927.622151:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2549 00000100:00080000:3.0:1775765927.622157:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765927.622161:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765927.622168:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765927.622180:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967171584/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765927.622199:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765927/real 1775765927] req@ffff8f9970edca80 x1862022967171200/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765927.622219:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967171200/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765927.622230:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765927.622235:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765927.622240:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967171200:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765927.622261:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967171328:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765927.622281:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765927.622335:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765927/real 1775765927] req@ffff8f9970edea00 x1862022967171328/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765927.622346:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967171328/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765927.622354:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765927.622356:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765927.622359:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967171328:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765927.622364:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967171456:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765927.622380:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765927.622387:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765927/real 1775765927] req@ffff8f9970edf800 x1862022967171456/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765927.622397:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967171456/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765927.622403:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765927.622406:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765927.622408:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967171456:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765927.622413:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967171584:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765927.622425:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765927.622432:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765927/real 1775765927] req@ffff8f9970edd880 x1862022967171584/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765927.622441:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967171584/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765927.622461:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765927.622463:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765927.622466:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967171584:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765928.517862:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775765928.517980:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765929.541926:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765929.541966:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765930.565812:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765930.565927:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765931.589840:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765931.589900:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765932.613875:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765932.614025:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765932.741957:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765932.741969:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765932.741975:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765932.741987:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2554 00000100:00080000:0.0:1775765932.742012:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765932.742019:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765932.742211:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967171712/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765932.742265:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765932.742271:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765932.742274:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765932.742278:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2554 00000100:00080000:0.0:1775765932.742289:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765932.742295:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765932.742306:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765932.742308:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967171712:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765932.742346:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967171840/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765932.742360:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765932.742364:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765932.742366:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765932.742371:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2554 00000100:00080000:0.0:1775765932.742388:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775765932.742408:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775765932.742412:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765932.742419:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765932.742446:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967171968/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765932.742454:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775765932.742458:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765932/real 1775765932] req@ffff8f9970a77100 x1862022967171712/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765932.742459:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765932.742462:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765932.742470:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2554 00000100:00080000:0.0:1775765932.742476:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765932.742479:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765932.742484:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765932.742486:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967171712/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765932.742495:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967172096/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765932.742503:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765932.742511:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765932.742517:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967171712:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765932.742543:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967171840:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765932.742564:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765932.742574:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765932/real 1775765932] req@ffff8f9970a76300 x1862022967171840/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765932.742583:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967171840/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765932.742609:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765932.742611:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765932.742614:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967171840:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765932.742618:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967171968:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765932.742633:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765932.742640:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765932/real 1775765932] req@ffff8f9970a75180 x1862022967171968/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765932.742648:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967171968/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765932.742656:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765932.742660:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765932.742663:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967171968:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765932.742669:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967172096:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765932.742690:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765932.742722:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765932/real 1775765932] req@ffff8f9970a76a00 x1862022967172096/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765932.742733:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967172096/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765932.742769:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765932.742772:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765932.742787:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967172096:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765933.637864:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765933.637870:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765934.661909:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765934.661985:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765935.685904:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765935.686054:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765936.709942:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765936.709957:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765937.733843:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765937.733928:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765937.862111:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765937.862126:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765937.862133:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765937.862149:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2559 00000100:00080000:2.0:1775765937.862166:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765937.862175:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765937.862247:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1500 x1862022967172224/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765937.862327:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765937.862332:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765937.862339:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765937.862346:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2559 00000100:00080000:2.0:1775765937.862352:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765937.862361:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765937.862376:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765937.862393:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0700 x1862022967172352/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765937.862409:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765937.862414:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765937.862864:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765937.862879:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2559 00000100:00080000:2.0:1775765937.862891:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765937.862898:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765937.862910:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765937.862944:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0e00 x1862022967172480/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765937.862944:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967172224:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775765937.862976:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765937.862980:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765937.862982:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765937.862988:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2559 00000100:00080000:2.0:1775765937.862991:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765937.862997:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765937.863005:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765937.863017:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2d80 x1862022967172608/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775765937.863142:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765937.863187:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765937/real 1775765937] req@ffff8f99703d1500 x1862022967172224/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765937.863205:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1500 x1862022967172224/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765937.863214:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765937.863219:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765937.863229:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967172224:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765937.863241:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967172352:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775765937.863289:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765937.863308:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765937/real 1775765937] req@ffff8f99703d0700 x1862022967172352/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765937.863330:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0700 x1862022967172352/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765937.863339:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765937.863343:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765937.863347:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967172352:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775765937.863372:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967172480:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775765937.863446:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765937.863455:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765937/real 1775765937] req@ffff8f99703d0e00 x1862022967172480/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765937.863463:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0e00 x1862022967172480/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765937.863468:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765937.863471:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765937.863473:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967172480:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775765937.863582:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967172608:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775765937.863599:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765937.863642:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765937/real 1775765937] req@ffff8f99703d2d80 x1862022967172608/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765937.863673:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2d80 x1862022967172608/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765937.863690:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765937.863696:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765937.863699:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967172608:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765938.757797:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765938.757868:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765939.781832:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765939.781877:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765940.805805:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765940.805880:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765941.829856:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765941.829934:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765942.853832:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765942.853927:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765942.981818:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765942.981826:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765942.981830:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765942.981838:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2564 00000100:00080000:3.0:1775765942.981847:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765942.981853:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765942.981885:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967172736/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765942.981910:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765942.981913:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765942.981914:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765942.981917:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2564 00000100:00080000:3.0:1775765942.981920:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765942.981923:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765942.981931:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765942.981941:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967172736:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765942.981942:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967172864/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765942.981955:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765942.981957:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765942.981959:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765942.981961:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2564 00000100:00080000:3.0:1775765942.981963:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765942.981966:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765942.981969:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765942.981976:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967172992/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765942.981982:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765942.981985:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765942.981986:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765942.981988:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2564 00000100:00080000:3.0:1775765942.981993:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765942.981995:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765942.981998:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765942.982006:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967173120/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765942.982010:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765942.982068:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765942/real 1775765942] req@ffff8f9970edf480 x1862022967172736/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765942.982114:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967172736/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765942.982123:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765942.982126:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765942.982129:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967172736:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765942.982159:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967172864:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765942.982175:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765942.982184:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765942/real 1775765942] req@ffff8f9970ede300 x1862022967172864/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765942.982192:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967172864/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765942.982198:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765942.982200:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765942.982202:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967172864:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765942.982206:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967172992:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765942.982217:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765942.982223:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765942/real 1775765942] req@ffff8f9970edc000 x1862022967172992/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765942.982230:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967172992/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765942.982251:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765942.982253:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765942.982256:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967172992:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765942.982259:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967173120:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765942.982278:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765942.982290:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765942/real 1775765942] req@ffff8f9970ede680 x1862022967173120/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765942.982302:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967173120/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765942.982311:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765942.982314:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765942.982317:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967173120:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765943.877872:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765943.877886:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765944.901830:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765944.901906:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765945.925898:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765945.925991:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765946.949873:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765946.950039:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765947.973806:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765947.973877:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765948.101854:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765948.101865:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765948.101977:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765948.101989:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2569 00000100:00080000:1.0:1775765948.102002:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765948.102010:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765948.102057:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967173248/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765948.102097:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765948.102101:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765948.102103:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765948.102107:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2569 00000100:00080000:1.0:1775765948.102110:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765948.102113:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765948.102122:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765948.102159:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967173376/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765948.102159:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967173248:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775765948.102188:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765948.102192:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765948.102193:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765948.102198:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2569 00000100:00080000:1.0:1775765948.102203:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765948.102207:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765948.102211:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765948.102223:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967173504/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765948.102251:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765948.102255:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765948.102257:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765948.102261:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2569 00000100:00080000:1.0:1775765948.102264:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765948.102274:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:2.0:1775765948.102278:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:1.0:1775765948.102282:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765948.102294:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967173632/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765948.102354:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765948/real 1775765948] req@ffff8f994f2e6d80 x1862022967173248/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765948.102380:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967173248/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765948.102478:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765948.102484:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765948.102488:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967173248:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765948.102521:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967173376:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765948.102638:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765948.102680:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765948/real 1775765948] req@ffff8f994f2e4a80 x1862022967173376/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765948.102693:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967173376/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765948.102700:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765948.102703:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765948.102706:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967173376:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765948.102711:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967173504:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765948.102748:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765948.102764:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765948/real 1775765948] req@ffff8f994f2e7800 x1862022967173504/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765948.102865:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967173504/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765948.102879:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765948.102883:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765948.102887:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967173504:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765948.102912:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967173632:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765948.102938:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765948.102949:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765948/real 1775765948] req@ffff8f994f2e5180 x1862022967173632/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765948.102960:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967173632/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765948.102968:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765948.102971:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765948.102974:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967173632:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765948.997848:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765948.997919:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765950.021833:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765950.021915:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765951.045931:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765951.045936:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765952.069869:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765952.069961:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765953.093829:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765953.093837:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765953.221889:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765953.221902:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765953.221908:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765953.221922:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2574 00000100:00080000:2.0:1775765953.221935:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765953.221945:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765953.221992:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3800 x1862022967173760/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765953.222034:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765953.222040:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765953.222103:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765953.222109:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2574 00000100:00080000:2.0:1775765953.222114:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765953.222119:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765953.222130:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765953.222137:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967173760:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765953.222147:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2680 x1862022967173888/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765953.222161:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765953.222165:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765953.222167:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765953.222172:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2574 00000100:00080000:2.0:1775765953.222188:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765953.222192:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:3.0:1775765953.222197:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:2.0:1775765953.222199:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765953.222216:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0000 x1862022967174016/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765953.222228:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765953.222232:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765953.222234:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765953.222239:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2574 00000100:00100000:3.0:1775765953.222242:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765953/real 1775765953] req@ffff8f99703d3800 x1862022967173760/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765953.222246:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765953.222256:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765953.222263:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765953.222265:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3800 x1862022967173760/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765953.222277:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765953.222284:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765953.222286:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0380 x1862022967174144/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765953.222288:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967173760:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765953.222312:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967173888:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775765953.222328:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765953.222336:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765953/real 1775765953] req@ffff8f99703d2680 x1862022967173888/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765953.222344:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2680 x1862022967173888/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765953.222349:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765953.222351:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765953.222353:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967173888:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775765953.222357:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967174016:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775765953.222367:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765953.222373:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765953/real 1775765953] req@ffff8f99703d0000 x1862022967174016/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765953.222380:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0000 x1862022967174016/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765953.222384:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765953.222386:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765953.222388:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967174016:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775765953.222391:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967174144:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775765953.222402:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765953.222408:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765953/real 1775765953] req@ffff8f99703d0380 x1862022967174144/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765953.222415:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0380 x1862022967174144/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765953.222419:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765953.222421:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765953.222423:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967174144:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765954.117853:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765954.117904:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765955.141821:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765955.141871:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765956.165830:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765956.165888:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765957.189850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765957.189920:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765958.213914:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765958.213926:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765958.341820:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765958.341829:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765958.341834:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765958.341842:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2579 00000100:00080000:2.0:1775765958.341852:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765958.341857:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765958.341899:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0380 x1862022967174272/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765958.341929:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765958.341933:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765958.341935:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765958.341940:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2579 00000100:00080000:2.0:1775765958.341944:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765958.341948:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765958.341957:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765958.341972:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0000 x1862022967174400/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765958.341984:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765958.341989:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1775765958.341990:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967174272:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775765958.341992:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765958.342002:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2579 00000100:00080000:2.0:1775765958.342007:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765958.342011:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765958.342017:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765958.342041:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2680 x1862022967174528/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765958.342052:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765958.342056:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765958.342058:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765958.342062:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2579 00000800:00000100:3.0:1775765958.342071:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775765958.342076:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765958.342082:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765958.342088:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765958.342103:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3800 x1862022967174656/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765958.342134:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765958/real 1775765958] req@ffff8f99703d0380 x1862022967174272/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765958.342155:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0380 x1862022967174272/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765958.342165:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765958.342170:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765958.342174:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967174272:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765958.342196:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967174400:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775765958.342218:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765958.342229:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765958/real 1775765958] req@ffff8f99703d0000 x1862022967174400/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765958.342249:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0000 x1862022967174400/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765958.342257:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765958.342261:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765958.342264:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967174400:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775765958.342270:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967174528:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775765958.342288:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765958.342297:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765958/real 1775765958] req@ffff8f99703d2680 x1862022967174528/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765958.342309:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2680 x1862022967174528/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765958.342317:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765958.342320:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765958.342323:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967174528:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775765958.342328:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967174656:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775765958.342344:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765958.342353:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765958/real 1775765958] req@ffff8f99703d3800 x1862022967174656/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765958.342365:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3800 x1862022967174656/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765958.342379:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765958.342383:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765958.342386:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967174656:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765959.237806:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765959.237868:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765960.261846:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765960.261912:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765961.285819:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765961.285896:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765962.309812:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765962.309908:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765963.333878:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765963.333925:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00080000:2.0:1775765963.461860:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765963.461871:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765963.462000:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765963.462010:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2585 00000100:00080000:2.0:1775765963.462020:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765963.462026:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765963.462058:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2d80 x1862022967174784/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765963.462084:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765963.462087:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765963.462089:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765963.462092:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2585 00000100:00080000:2.0:1775765963.462094:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765963.462096:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765963.462103:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765963.462110:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0e00 x1862022967174912/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765963.462118:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765963.462120:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765963.462122:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765963.462125:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2585 00000100:00080000:2.0:1775765963.462127:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765963.462129:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765963.462132:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765963.462138:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0700 x1862022967175040/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765963.462147:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967174784:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775765963.462172:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765963.462175:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765963.462176:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765963.462179:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2585 00000100:00080000:2.0:1775765963.462182:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765963.462184:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775765963.462187:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765963.462195:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1500 x1862022967175168/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775765963.462214:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765963.462256:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765963/real 1775765963] req@ffff8f99703d2d80 x1862022967174784/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765963.462274:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2d80 x1862022967174784/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765963.462286:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765963.462291:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765963.462303:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967174784:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765963.462312:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967174912:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775765963.462334:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765963.462343:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765963/real 1775765963] req@ffff8f99703d0e00 x1862022967174912/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765963.462352:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0e00 x1862022967174912/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765963.462357:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765963.462360:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765963.462362:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967174912:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775765963.462381:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967175040:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775765963.462419:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765963.462431:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765963/real 1775765963] req@ffff8f99703d0700 x1862022967175040/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765963.462445:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0700 x1862022967175040/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765963.462453:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765963.462456:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765963.462458:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967175040:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775765963.462462:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967175168:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775765963.462477:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765963.462483:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765963/real 1775765963] req@ffff8f99703d1500 x1862022967175168/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765963.462491:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1500 x1862022967175168/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765963.462495:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765963.462497:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765963.462499:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967175168:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765964.357899:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765964.357920:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765965.381865:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765965.381872:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775765966.405898:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765966.405995:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765967.429842:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765967.429913:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765968.453875:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765968.453944:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765968.581852:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765968.581860:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765968.581864:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765968.581874:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2590 00000100:00080000:0.0:1775765968.581885:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765968.581891:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765968.581924:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967175296/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765968.581951:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765968.581954:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765968.581955:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765968.581959:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2590 00000100:00080000:0.0:1775765968.581961:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765968.581963:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765968.581971:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765968.581981:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967175424/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765968.581988:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765968.581990:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765968.581991:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765968.581993:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2590 00000100:00080000:0.0:1775765968.581995:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765968.581997:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765968.582001:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765968.582008:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967175552/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765968.582015:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765968.582017:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765968.582018:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765968.582021:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2590 00000100:00080000:0.0:1775765968.582024:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775765968.582026:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967175296:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775765968.582028:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765968.582034:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765968.582073:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967175680/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775765968.582088:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765968.582136:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765968/real 1775765968] req@ffff8f9970a77800 x1862022967175296/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765968.582153:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967175296/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765968.582160:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765968.582163:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765968.582166:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967175296:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765968.582171:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967175424:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775765968.582186:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765968.582196:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765968/real 1775765968] req@ffff8f9970a77480 x1862022967175424/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765968.582205:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967175424/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765968.582211:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765968.582213:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765968.582215:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967175424:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775765968.582218:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967175552:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775765968.582229:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765968.582234:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765968/real 1775765968] req@ffff8f9970a75500 x1862022967175552/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765968.582241:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967175552/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765968.582251:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765968.582253:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765968.582255:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967175552:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775765968.582278:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967175680:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775765968.582296:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775765968.582306:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765968/real 1775765968] req@ffff8f9970a74380 x1862022967175680/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765968.582319:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967175680/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765968.582328:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765968.582331:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775765968.582335:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967175680:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765969.477815:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765969.477934:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765970.501858:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765970.501930:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765971.525834:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765971.525936:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765972.549935:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765972.549938:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765973.573879:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765973.573939:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765973.701863:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765973.701874:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765973.701881:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765973.701894:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2595 00000100:00080000:0.0:1775765973.701908:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765973.701916:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765973.701970:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967175808/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765973.701998:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765973.702002:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765973.702003:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765973.702007:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2595 00000100:00080000:0.0:1775765973.702010:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765973.702013:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765973.702024:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765973.702035:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967175936/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765973.702043:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765973.702046:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765973.702049:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765973.702051:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967175808:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775765973.702054:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2595 00000100:00080000:0.0:1775765973.702060:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765973.702064:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765973.702067:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765973.702077:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967176064/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765973.702139:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775765973.702181:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775765973.702182:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765973/real 1775765973] req@ffff8f9970a75c00 x1862022967175808/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765973.702195:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765973.702202:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765973.702215:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2595 00000100:00100000:2.0:1775765973.702220:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967175808/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765973.702235:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765973.702243:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765973.702254:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1775765973.702277:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1775765973.702282:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967176192/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765973.702284:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765973.702301:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967175808:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765973.702361:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967175936:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765973.702387:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765973.702399:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765973/real 1775765973] req@ffff8f9970a74700 x1862022967175936/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765973.702409:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967175936/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765973.702415:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765973.702417:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765973.702419:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967175936:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765973.702425:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967176064:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765973.702439:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765973.702446:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765973/real 1775765973] req@ffff8f9970a76d80 x1862022967176064/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765973.702454:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967176064/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765973.702459:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765973.702461:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765973.702463:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967176064:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765973.702483:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967176192:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765973.702499:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765973.702512:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765973/real 1775765973] req@ffff8f9970a74380 x1862022967176192/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765973.702535:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967176192/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765973.702542:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765973.702548:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765973.702551:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967176192:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765974.597832:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765974.597887:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765975.621871:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765975.621954:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765976.645843:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765976.645909:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765977.669881:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765977.669899:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765978.693832:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765978.694062:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765978.821888:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765978.821906:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765978.821915:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765978.821931:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2600 00000100:00080000:0.0:1775765978.821949:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765978.821962:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765978.822022:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967176320/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765978.822065:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765978.822070:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765978.822073:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765978.822080:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2600 00000100:00080000:0.0:1775765978.822087:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765978.822095:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765978.822109:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765978.822124:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967176320:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765978.822133:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967176448/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765978.822153:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765978.822158:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765978.822161:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765978.822168:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2600 00000100:00080000:0.0:1775765978.822173:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765978.822177:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765978.822183:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765978.822200:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967176576/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765978.822211:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765978.822216:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765978.822220:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765978.822224:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2600 00000100:00080000:0.0:1775765978.822229:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765978.822234:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765978.822242:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765978.822281:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967176704/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765978.822338:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967176448:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765978.822423:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967176576:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765978.822456:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967176704:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765979.717916:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765979.717933:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765980.741917:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765980.742033:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765981.765811:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765981.765952:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1775765981.894045:0:2367:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:0.0:1775765981.894142:0:2367:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1775765981.894165:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775765981.894218:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775765981.894293:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775765981.894309:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:2.0:1775765981.894348:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765978/real 1775765981] req@ffff8f9970a77480 x1862022967176448/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765981.894451:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967176448/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765981.894467:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765981.894474:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765981.894480:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967176448:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765981.894491:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765978/real 1775765981] req@ffff8f9970a77800 x1862022967176576/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765981.894519:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967176576/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765981.894528:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765981.894537:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765981.894541:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967176576:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765981.894549:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765978/real 1775765981] req@ffff8f9970a76a00 x1862022967176704/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765981.894563:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967176704/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765981.894582:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765981.894586:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765981.894589:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967176704:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765981.894597:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765978/real 1775765981] req@ffff8f9970a75500 x1862022967176320/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765981.894634:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967176320/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765981.894645:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765981.894650:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765981.894653:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967176320:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765982.789852:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765982.789933:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765983.813836:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765983.813930:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765983.941867:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765983.941878:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765983.941882:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765983.941893:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2605 00000100:00080000:0.0:1775765983.941906:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765983.941913:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775765983.941968:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967176832/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765983.942004:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765983.942009:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765983.942011:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765983.942016:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2605 00000100:00080000:0.0:1775765983.942022:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765983.942027:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765983.942039:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765983.942048:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967176832:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775765983.942053:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967176960/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765983.942063:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765983.942068:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765983.942069:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765983.942072:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2605 00000100:00080000:0.0:1775765983.942076:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765983.942079:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775765983.942084:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765983.942097:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967177088/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765983.942108:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765983.942112:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765983.942114:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765983.942119:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2605 00000100:00080000:0.0:1775765983.942124:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765983.942128:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:2.0:1775765983.942134:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:0.0:1775765983.942138:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765983.942156:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967177216/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765983.942188:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765983/real 1775765983] req@ffff8f9970a75180 x1862022967176832/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765983.942227:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967176832/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765983.942240:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765983.942246:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765983.942252:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967176832:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765983.942285:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967176960:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765983.942311:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765983.942332:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765983/real 1775765983] req@ffff8f9970a76300 x1862022967176960/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765983.942344:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967176960/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765983.942386:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765983.942390:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765983.942395:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967176960:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765983.942402:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967177088:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765983.942427:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765983.942439:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765983/real 1775765983] req@ffff8f9970a77100 x1862022967177088/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765983.942452:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967177088/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765983.942461:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765983.942465:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765983.942468:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967177088:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765983.942475:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967177216:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765983.942495:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765983.942505:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765983/real 1775765983] req@ffff8f9970a75500 x1862022967177216/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765983.942516:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967177216/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765983.942524:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765983.942527:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765983.942548:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967177216:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765984.837869:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765984.837932:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765985.861878:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765985.861939:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765986.885932:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765986.885981:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765987.909861:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765987.909919:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765988.933802:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765988.933897:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765989.061869:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765989.061877:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765989.061881:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765989.061890:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2610 00000100:00080000:1.0:1775765989.061901:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765989.061907:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775765989.061940:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967177344/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765989.061963:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765989.061966:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765989.061967:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765989.061970:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2610 00000100:00080000:1.0:1775765989.061972:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765989.061975:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765989.062004:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967177344:192.168.202.146@tcp:250:kworker.0 00000008:00100000:1.0:1775765989.062018:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765989.062033:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967177472/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765989.062045:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765989.062048:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765989.062051:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765989.062055:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2610 00000100:00080000:1.0:1775765989.062060:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775765989.062064:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775765989.062068:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775765989.062075:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765989.062089:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967177600/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765989.062100:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765989.062103:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765989.062105:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765989.062107:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765989/real 1775765989] req@ffff8f994f2e4000 x1862022967177344/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765989.062113:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2610 00000100:00080000:1.0:1775765989.062122:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765989.062127:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775765989.062130:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967177344/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1775765989.062136:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1775765989.062139:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765989.062143:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765989.062147:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967177344:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775765989.062152:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967177728/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765989.062189:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967177472:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765989.062216:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765989.062229:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765989/real 1775765989] req@ffff8f994f2e6a00 x1862022967177472/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765989.062243:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967177472/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765989.062249:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765989.062251:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765989.062288:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967177472:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765989.062293:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967177600:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765989.062310:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765989.062318:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765989/real 1775765989] req@ffff8f994f2e5f80 x1862022967177600/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765989.062331:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967177600/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765989.062339:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765989.062342:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765989.062346:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967177600:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765989.062352:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967177728:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765989.062372:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765989.062384:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765989/real 1775765989] req@ffff8f994f2e6680 x1862022967177728/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765989.062393:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967177728/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765989.062399:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765989.062401:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765989.062403:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967177728:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765989.957838:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765989.957923:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765990.981862:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765990.981936:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765992.005843:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765992.005939:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765993.029850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765993.029975:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765994.053878:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765994.053888:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765994.181862:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765994.181874:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765994.181879:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765994.181891:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2615 00000100:00080000:3.0:1775765994.181903:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765994.181910:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765994.181955:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967177856/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765994.181981:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765994.181986:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765994.181988:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765994.181993:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2615 00000100:00080000:3.0:1775765994.181996:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765994.181999:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765994.182009:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765994.182019:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967177984/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765994.182028:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765994.182031:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765994.182032:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765994.182035:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2615 00000100:00080000:3.0:1775765994.182037:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765994.182040:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765994.182044:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765994.182053:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967178112/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765994.182056:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967177856:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775765994.182061:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765994.182068:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765994.182069:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765994.182072:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2615 00000100:00080000:3.0:1775765994.182073:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765994.182075:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765994.182079:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765994.182087:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967178240/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765994.182161:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765994.182214:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765994/real 1775765994] req@ffff8f9970edc700 x1862022967177856/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765994.182232:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967177856/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765994.182242:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765994.182246:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765994.182249:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967177856:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765994.182255:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967177984:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765994.182275:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765994.182287:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765994/real 1775765994] req@ffff8f9970eded80 x1862022967177984/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765994.182300:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967177984/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765994.182309:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765994.182313:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765994.182317:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967177984:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765994.182347:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967178112:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765994.182533:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765994.182571:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765994/real 1775765994] req@ffff8f9970edce00 x1862022967178112/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765994.182593:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967178112/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765994.182605:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765994.182610:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765994.182615:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967178112:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765994.182627:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967178240:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765994.182650:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765994.182663:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765994/real 1775765994] req@ffff8f9970edd500 x1862022967178240/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765994.182677:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967178240/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765994.182685:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765994.182689:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765994.182693:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967178240:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765995.077885:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765995.077996:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765996.101941:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765996.102060:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765997.125874:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765997.125931:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765998.149875:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775765998.149971:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765999.173843:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775765999.173974:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765999.301834:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765999.301843:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765999.301847:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765999.301857:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2620 00000100:00080000:3.0:1775765999.301868:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765999.301875:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775765999.301914:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddc00 x1862022967178368/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765999.301944:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765999.301947:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765999.301949:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765999.301953:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2620 00000100:00080000:3.0:1775765999.301957:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765999.301960:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765999.301970:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765999.301980:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967178368:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775765999.301982:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf100 x1862022967178496/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765999.301992:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765999.301995:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765999.301996:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765999.301999:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2620 00000100:00080000:3.0:1775765999.302002:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765999.302005:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765999.302009:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765999.302018:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967178624/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765999.302028:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765999.302033:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765999.302034:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765999.302037:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2620 00000100:00080000:3.0:1775765999.302040:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765999.302043:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775765999.302046:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765999.302084:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967178752/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765999.302119:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765999.302156:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765999/real 1775765999] req@ffff8f9970eddc00 x1862022967178368/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765999.302171:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddc00 x1862022967178368/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765999.302179:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765999.302181:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765999.302184:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967178368:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775765999.302203:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967178496:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775765999.302218:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765999.302226:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765999/real 1775765999] req@ffff8f9970edf100 x1862022967178496/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765999.302234:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf100 x1862022967178496/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765999.302239:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765999.302241:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765999.302243:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967178496:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775765999.302246:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967178624:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765999.302257:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765999.302263:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765999/real 1775765999] req@ffff8f9970edd500 x1862022967178624/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765999.302290:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967178624/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765999.302296:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765999.302299:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765999.302301:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967178624:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775765999.302305:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967178752:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775765999.302317:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775765999.302323:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765999/real 1775765999] req@ffff8f9970edce00 x1862022967178752/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765999.302331:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967178752/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765999.302336:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765999.302338:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775765999.302339:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967178752:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766000.197896:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766000.197982:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766001.221929:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766001.222035:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766002.245826:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766002.245905:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766003.269852:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766003.269920:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766004.293817:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766004.293902:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766004.421924:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766004.421937:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766004.421942:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766004.421956:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2626 00000100:00080000:3.0:1775766004.421971:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766004.421979:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766004.422056:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967178880/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766004.422097:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766004.422105:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766004.422107:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766004.422112:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2626 00000100:00080000:3.0:1775766004.422118:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766004.422123:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766004.422135:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766004.422152:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967179008/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766004.422168:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775766004.422170:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967178880:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775766004.422173:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766004.422177:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766004.422186:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2626 00000100:00080000:3.0:1775766004.422191:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766004.422196:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766004.422202:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766004.422216:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967179136/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766004.422226:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766004.422231:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766004.422233:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766004.422237:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2626 00000100:00080000:3.0:1775766004.422241:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766004.422246:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766004.422254:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766004.422279:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967179264/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766004.422296:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766004.422357:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766004/real 1775766004] req@ffff8f9970eded80 x1862022967178880/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766004.422384:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967178880/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766004.422397:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766004.422402:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766004.422407:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967178880:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766004.422439:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967179008:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775766004.422468:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766004.422484:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766004/real 1775766004] req@ffff8f9970edc700 x1862022967179008/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766004.422499:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967179008/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766004.422509:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766004.422513:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766004.422517:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967179008:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775766004.422525:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967179136:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766004.422550:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766004.422563:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766004/real 1775766004] req@ffff8f9970ede680 x1862022967179136/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766004.422576:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967179136/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766004.422593:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766004.422597:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766004.422601:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967179136:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766004.422608:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967179264:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766004.422626:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766004.422644:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766004/real 1775766004] req@ffff8f9970edc000 x1862022967179264/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766004.422657:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967179264/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766004.422665:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766004.422669:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766004.422672:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967179264:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766005.317859:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766005.317920:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766006.341828:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766006.342025:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766007.365819:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766007.365888:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766008.389910:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766008.389918:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766009.413868:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766009.414022:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766009.541899:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766009.541908:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766009.541912:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766009.541920:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2631 00000100:00080000:1.0:1775766009.541932:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766009.541941:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766009.541997:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967179392/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766009.542027:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766009.542030:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766009.542031:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766009.542034:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2631 00000100:00080000:1.0:1775766009.542037:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766009.542041:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766009.542074:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967179392:192.168.202.146@tcp:250:kworker.0 00000008:00100000:1.0:1775766009.542096:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766009.542113:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967179520/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766009.542126:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766009.542128:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766009.542131:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766009.542135:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2631 00000100:00080000:1.0:1775766009.542139:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766009.542143:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766009.542149:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766009.542156:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967179648/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766009.542164:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766009.542169:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766009.542171:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766009.542173:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2631 00000100:00080000:1.0:1775766009.542175:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766009.542177:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766009.542180:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766009.542190:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967179776/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766009.542190:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766009.542321:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766009/real 1775766009] req@ffff8f994f2e4700 x1862022967179392/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766009.542347:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967179392/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766009.542357:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766009.542361:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766009.542364:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967179392:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766009.542389:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967179520:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766009.542419:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766009.542429:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766009/real 1775766009] req@ffff8f994f2e6300 x1862022967179520/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766009.542438:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967179520/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766009.542443:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766009.542446:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766009.542448:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967179520:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766009.542452:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967179648:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766009.542465:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766009.542471:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766009/real 1775766009] req@ffff8f994f2e4e00 x1862022967179648/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766009.542479:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967179648/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766009.542484:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766009.542486:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766009.542488:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967179648:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766009.542492:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967179776:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766009.542503:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766009.542508:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766009/real 1775766009] req@ffff8f994f2e7b80 x1862022967179776/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766009.542515:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967179776/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766009.542521:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766009.542523:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766009.542524:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967179776:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766010.437911:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766010.437983:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766011.461840:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766011.461937:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766012.486015:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766012.486133:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766013.509866:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766013.509957:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766014.533870:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766014.533973:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766014.661903:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766014.661940:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766014.661945:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766014.661974:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2636 00000100:00080000:3.0:1775766014.661987:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766014.661995:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766014.662030:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967179904/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766014.662061:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766014.662065:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766014.662066:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766014.662070:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2636 00000100:00080000:3.0:1775766014.662072:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766014.662075:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766014.662091:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766014.662105:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967180032/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766014.662114:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967179904:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775766014.662124:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766014.662128:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766014.662130:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766014.662136:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2636 00000100:00080000:3.0:1775766014.662139:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766014.662142:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766014.662147:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766014.662168:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967180160/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766014.662176:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766014.662178:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766014.662180:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766014.662183:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2636 00000100:00080000:3.0:1775766014.662185:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766014.662188:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766014.662191:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766014.662198:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766014.662201:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967180288/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766014.662245:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766014/real 1775766014] req@ffff8f9970ede300 x1862022967179904/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766014.662260:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967179904/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766014.662273:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766014.662276:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766014.662280:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967179904:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766014.662301:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967180032:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775766014.662317:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766014.662330:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766014/real 1775766014] req@ffff8f9970edf480 x1862022967180032/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766014.662343:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967180032/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766014.662349:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766014.662359:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766014.662361:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967180032:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775766014.662366:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967180160:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766014.662378:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766014.662384:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766014/real 1775766014] req@ffff8f9970edd880 x1862022967180160/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766014.662392:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967180160/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766014.662397:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766014.662399:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766014.662401:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967180160:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766014.662405:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967180288:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766014.662415:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766014.662420:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766014/real 1775766014] req@ffff8f9970edf800 x1862022967180288/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766014.662427:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967180288/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766014.662432:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766014.662434:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766014.662436:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967180288:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766015.557870:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766015.557887:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766016.581886:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766016.582016:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766017.605812:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766017.605878:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766018.629842:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766018.629918:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766019.653902:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766019.654040:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766019.781822:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766019.781831:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766019.781836:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766019.781847:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2641 00000100:00080000:3.0:1775766019.781858:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766019.781878:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766019.781917:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967180416/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766019.781941:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766019.781944:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766019.781945:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766019.781948:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2641 00000100:00080000:3.0:1775766019.781951:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766019.781954:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766019.781962:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766019.781971:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967180544/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766019.781983:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766019.781986:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766019.781988:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775766019.781990:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967180416:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775766019.781993:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2641 00000100:00080000:3.0:1775766019.781999:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766019.782001:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766019.782005:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766019.782015:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967180672/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766019.782021:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766019.782024:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766019.782025:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766019.782027:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2641 00000100:00080000:3.0:1775766019.782030:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766019.782032:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766019.782035:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766019.782042:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967180800/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766019.782052:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766019.782094:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766019/real 1775766019] req@ffff8f9970edea00 x1862022967180416/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766019.782110:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967180416/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766019.782118:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766019.782122:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766019.782125:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967180416:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766019.782145:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967180544:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775766019.782160:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766019.782168:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766019/real 1775766019] req@ffff8f9970edca80 x1862022967180544/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766019.782176:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967180544/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766019.782189:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766019.782192:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766019.782194:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967180544:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775766019.782198:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967180672:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766019.782210:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766019.782216:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766019/real 1775766019] req@ffff8f9970edc380 x1862022967180672/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766019.782223:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967180672/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766019.782228:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766019.782231:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766019.782233:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967180672:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766019.782238:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967180800:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766019.782264:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766019.782276:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766019/real 1775766019] req@ffff8f9970edfb80 x1862022967180800/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766019.782317:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967180800/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766019.782323:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766019.782325:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766019.782327:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967180800:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766020.677820:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766020.677885:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766021.701824:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766021.701924:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766022.725887:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766022.725888:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775766023.749844:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766023.750031:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766024.773871:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766024.773946:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766024.901917:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766024.901926:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766024.901931:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766024.901939:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2646 00000100:00080000:0.0:1775766024.901951:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766024.901957:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766024.901992:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967180928/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766024.902021:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766024.902024:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766024.902026:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766024.902029:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2646 00000100:00080000:0.0:1775766024.902032:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766024.902035:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766024.902043:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766024.902077:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967180928:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766024.902098:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967181056/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766024.902108:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766024.902110:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766024.902112:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766024.902115:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2646 00000100:00080000:0.0:1775766024.902118:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766024.902121:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766024.902126:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766024.902133:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967181184/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766024.902141:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766024.902143:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766024.902145:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766024.902147:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2646 00000100:00080000:0.0:1775766024.902150:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766024.902155:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:3.0:1775766024.902158:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:0.0:1775766024.902160:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766024.902169:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967181312/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766024.902225:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766024/real 1775766024] req@ffff8f9970a76a00 x1862022967180928/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766024.902257:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967180928/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766024.902269:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766024.902274:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766024.902280:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967180928:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766024.902329:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967181056:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766024.902358:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766024.902373:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766024/real 1775766024] req@ffff8f9970a77800 x1862022967181056/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766024.902388:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967181056/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766024.902398:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766024.902402:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766024.902406:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967181056:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766024.902414:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967181184:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766024.902438:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766024.902452:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766024/real 1775766024] req@ffff8f9970a77480 x1862022967181184/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766024.902467:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967181184/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766024.902485:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766024.902488:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766024.902493:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967181184:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766024.902500:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967181312:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766024.902538:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766024.902550:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766024/real 1775766024] req@ffff8f9970a74380 x1862022967181312/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766024.902564:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967181312/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766024.902573:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766024.902577:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766024.902581:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967181312:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766025.797894:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766025.797926:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766026.821867:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766026.821940:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766027.845845:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766027.845934:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766028.869827:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766028.869926:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766029.893900:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766029.893920:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766030.021942:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766030.021956:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766030.021963:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766030.021989:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2651 00000100:00080000:3.0:1775766030.022000:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766030.022008:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766030.022045:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddf80 x1862022967181440/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766030.022076:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766030.022080:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766030.022082:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766030.022088:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2651 00000100:00080000:3.0:1775766030.022093:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766030.022098:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766030.022108:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766030.022110:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967181440:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766030.022151:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd180 x1862022967181568/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766030.022172:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766030.022178:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766030.022181:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766030.022187:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2651 00000100:00080000:3.0:1775766030.022212:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766030.022215:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766030.022221:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766030.022228:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766030.022237:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967181696/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766030.022248:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766030.022251:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766030.022253:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766030.022257:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2651 00000100:00080000:3.0:1775766030.022262:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775766030.022269:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766030/real 1775766030] req@ffff8f9970eddf80 x1862022967181440/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766030.022269:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766030.022279:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766030.022290:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddf80 x1862022967181440/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766030.022296:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967181824/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766030.022301:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766030.022310:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766030.022317:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967181440:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766030.022338:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967181568:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775766030.022355:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766030.022363:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766030/real 1775766030] req@ffff8f9970edd180 x1862022967181568/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766030.022371:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd180 x1862022967181568/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766030.022377:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766030.022380:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766030.022383:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967181568:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775766030.022389:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967181696:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766030.022408:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766030.022419:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766030/real 1775766030] req@ffff8f9970edfb80 x1862022967181696/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766030.022440:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967181696/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766030.022449:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766030.022451:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766030.022453:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967181696:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766030.022460:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967181824:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766030.022483:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766030.022493:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766030/real 1775766030] req@ffff8f9970edc380 x1862022967181824/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766030.022505:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967181824/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766030.022512:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766030.022515:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766030.022518:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967181824:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766030.917875:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766030.917937:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766031.941912:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766031.941985:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766032.965895:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766032.965908:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:3.0:1775766033.989879:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766033.989994:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766035.013866:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766035.013930:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766035.141854:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766035.141862:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766035.141867:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766035.141877:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2656 00000100:00080000:3.0:1775766035.141888:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766035.141895:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766035.141954:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967181952/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766035.141991:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766035.141995:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766035.141996:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766035.142000:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2656 00000100:00080000:3.0:1775766035.142003:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766035.142005:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766035.142013:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766035.142029:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967182080/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766035.142038:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766035.142041:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766035.142043:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766035.142046:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2656 00000100:00080000:3.0:1775766035.142049:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775766035.142052:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967181952:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775766035.142054:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766035.142063:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766035.142073:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967182208/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766035.142081:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766035.142084:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766035.142090:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766035.142093:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2656 00000100:00080000:3.0:1775766035.142097:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766035.142100:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766035.142103:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766035.142112:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967182336/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766035.142120:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766035.142168:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766035/real 1775766035] req@ffff8f9970edca80 x1862022967181952/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766035.142185:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967181952/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766035.142236:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766035.142241:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766035.142244:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967181952:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766035.142251:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967182080:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775766035.142273:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766035.142284:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766035/real 1775766035] req@ffff8f9970edea00 x1862022967182080/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766035.142294:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967182080/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766035.142300:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766035.142303:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766035.142305:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967182080:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775766035.142327:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967182208:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766035.142341:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766035.142348:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766035/real 1775766035] req@ffff8f9970edf800 x1862022967182208/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766035.142357:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967182208/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766035.142363:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766035.142365:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766035.142367:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967182208:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766035.142372:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967182336:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766035.142383:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766035.142390:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766035/real 1775766035] req@ffff8f9970edd880 x1862022967182336/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766035.142399:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967182336/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766035.142405:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766035.142407:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766035.142409:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967182336:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766036.037860:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766036.037915:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766037.061915:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766037.061937:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775766038.085862:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766038.085928:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766039.109836:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766039.109970:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766040.133864:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766040.133973:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766040.261819:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766040.261827:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766040.261832:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766040.261841:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2661 00000100:00080000:3.0:1775766040.261852:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766040.261860:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766040.261901:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967182464/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766040.261938:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766040.261941:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766040.261943:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766040.261946:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2661 00000100:00080000:3.0:1775766040.261950:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766040.261953:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766040.261961:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766040.261987:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967182592/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766040.261996:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766040.261999:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766040.262000:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766040.262003:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2661 00000100:00080000:3.0:1775766040.262005:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766040.262011:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766040.262013:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967182464:192.168.202.146@tcp:250:kworker.0 00000008:00100000:3.0:1775766040.262016:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766040.262028:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967182720/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766040.262037:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766040.262039:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766040.262041:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766040.262043:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2661 00000100:00080000:3.0:1775766040.262046:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766040.262049:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766040.262053:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766040.262060:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967182848/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766040.262097:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766040.262135:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766040/real 1775766040] req@ffff8f9970edf480 x1862022967182464/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766040.262150:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967182464/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766040.262158:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766040.262161:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766040.262165:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967182464:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766040.262170:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967182592:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775766040.262187:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766040.262195:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766040/real 1775766040] req@ffff8f9970ede300 x1862022967182592/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766040.262204:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967182592/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766040.262210:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766040.262212:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766040.262214:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967182592:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775766040.262235:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967182720:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766040.262247:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766040.262253:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766040/real 1775766040] req@ffff8f9970edc000 x1862022967182720/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766040.262261:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967182720/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766040.262267:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766040.262269:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766040.262270:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967182720:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766040.262274:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967182848:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766040.262295:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766040.262301:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766040/real 1775766040] req@ffff8f9970ede680 x1862022967182848/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766040.262309:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967182848/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766040.262315:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766040.262317:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766040.262319:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967182848:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766041.157836:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766041.157898:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766042.181820:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766042.181933:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766043.205842:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766043.206055:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766044.229944:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766044.229970:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766045.253813:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766045.253903:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766045.381871:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766045.381884:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766045.381890:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766045.381902:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2666 00000100:00080000:3.0:1775766045.381917:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766045.381927:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766045.381974:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967182976/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766045.382017:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766045.382024:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766045.382027:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766045.382035:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2666 00000100:00080000:3.0:1775766045.382043:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766045.382049:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766045.382057:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967182976:192.168.202.146@tcp:250:kworker.0 00000008:00100000:3.0:1775766045.382076:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766045.382111:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967183104/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766045.382128:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766045.382136:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766045.382139:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766045.382146:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2666 00000100:00080000:3.0:1775766045.382153:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766045.382159:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766045.382169:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766045.382191:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967183232/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766045.382207:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775766045.382293:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967183104:192.168.202.146@tcp:38:kworker.0 00000100:00080000:3.0:1775766045.382306:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766045.382310:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775766045.382366:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967183232:192.168.202.146@tcp:8:kworker.0 00000100:00080000:3.0:1775766045.382448:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2666 00000100:00080000:3.0:1775766045.382454:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766045.382458:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766045.382465:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766045.382479:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967183360/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766045.382543:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967183360:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766046.277825:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766046.278001:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766047.301811:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766047.301938:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766048.325859:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766048.325919:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775766048.454014:0:2369:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:1.0:1775766048.454074:0:2369:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775766048.454135:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775766048.454185:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775766048.454196:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775766048.454206:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:0.0:1775766048.454228:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766045/real 1775766048] req@ffff8f9970eded80 x1862022967183104/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766048.454260:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967183104/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766048.454272:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766048.454399:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766048.454404:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967183104:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775766048.454413:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766045/real 1775766048] req@ffff8f9970edce00 x1862022967183232/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766048.454427:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967183232/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766048.454435:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766048.454439:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766048.454442:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967183232:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766048.454449:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766045/real 1775766048] req@ffff8f9970edc700 x1862022967182976/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766048.454468:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967182976/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766048.454477:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766048.454481:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766048.454484:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967182976:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766048.454513:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766045/real 1775766048] req@ffff8f9970edd500 x1862022967183360/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766048.454528:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967183360/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766048.454537:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766048.454541:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766048.454544:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967183360:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766049.349817:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766049.349875:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766050.373839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766050.373906:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766050.501885:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766050.501893:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766050.501896:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766050.501904:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2672 00000100:00080000:0.0:1775766050.501915:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766050.501920:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766050.501958:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967183488/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766050.501986:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766050.501991:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766050.501992:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766050.501997:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2672 00000100:00080000:0.0:1775766050.502002:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766050.502006:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766050.502017:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766050.502033:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967183616/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766050.502034:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967183488:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775766050.502052:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766050.502055:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766050.502057:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766050.502061:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2672 00000100:00080000:0.0:1775766050.502065:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766050.502069:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766050.502075:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766050.502087:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967183744/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766050.502097:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766050.502100:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766050.502102:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766050.502105:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2672 00000100:00080000:0.0:1775766050.502109:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1775766050.502114:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775766050.502119:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766050.502126:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766050.502147:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967183872/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766050.502173:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766050/real 1775766050] req@ffff8f9970a76d80 x1862022967183488/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766050.502193:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967183488/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766050.502204:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766050.502208:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766050.502213:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967183488:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766050.502240:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967183616:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766050.502266:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766050.502281:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766050/real 1775766050] req@ffff8f9970a74700 x1862022967183616/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766050.502293:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967183616/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766050.502302:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766050.502305:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766050.502309:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967183616:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766050.502360:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967183744:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766050.502383:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766050.502395:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766050/real 1775766050] req@ffff8f9970a75c00 x1862022967183744/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766050.502407:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967183744/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766050.502415:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766050.502418:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766050.502421:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967183744:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766050.502426:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967183872:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766050.502452:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766050.502462:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766050/real 1775766050] req@ffff8f9970a74380 x1862022967183872/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766050.502473:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967183872/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766050.502481:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766050.502485:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766050.502488:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967183872:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766051.397903:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766051.397911:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766052.421812:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766052.421895:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766053.445873:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766053.445927:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766054.469895:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766054.469925:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766055.493938:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766055.493949:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766055.621997:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766055.622018:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766055.622027:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766055.622048:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2677 00000100:00080000:2.0:1775766055.622070:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766055.622085:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766055.622158:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1180 x1862022967184000/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766055.622211:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766055.622218:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766055.622222:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766055.622233:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2677 00000100:00080000:2.0:1775766055.622252:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766055.622264:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766055.622283:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766055.622289:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967184000:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766055.622332:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3100 x1862022967184128/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766055.622391:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766055.622467:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766055/real 1775766055] req@ffff8f99703d1180 x1862022967184000/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766055.622511:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1180 x1862022967184000/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766055.622552:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766055.622558:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766055.622562:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967184000:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775766055.622622:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766055.622628:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766055.622632:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766055.622640:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2677 00000100:00100000:1.0:1775766055.622648:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967184128:192.168.202.146@tcp:38:kworker.0 00000100:00080000:2.0:1775766055.622649:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766055.622663:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766055.622678:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775766055.622684:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766055.622700:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766055/real 1775766055] req@ffff8f99703d3100 x1862022967184128/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766055.622711:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3480 x1862022967184256/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766055.622744:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766055.622747:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766055.622749:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775766055.622770:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3100 x1862022967184128/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766055.622782:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766055.622787:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766055.622792:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967184128:192.168.202.146@tcp:38:kworker.0 00000100:00080000:2.0:1775766055.622797:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2677 00000100:00080000:2.0:1775766055.622804:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766055.622810:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766055.622814:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967184256:192.168.202.146@tcp:8:kworker.0 00000008:00100000:2.0:1775766055.622822:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766055.622862:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2300 x1862022967184384/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766055.622874:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766055.622896:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766055/real 1775766055] req@ffff8f99703d3480 x1862022967184256/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766055.622957:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3480 x1862022967184256/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766055.622965:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766055.622968:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766055.622971:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967184256:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766055.622982:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967184384:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766055.623000:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766055.623009:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766055/real 1775766055] req@ffff8f99703d2300 x1862022967184384/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766055.623018:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2300 x1862022967184384/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766055.623025:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766055.623027:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766055.623030:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967184384:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766056.517838:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766056.517963:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766057.541866:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766057.541964:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766058.565897:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766058.565930:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766059.589923:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766059.590046:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766060.613818:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766060.613884:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766060.741819:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766060.741828:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766060.741832:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766060.741841:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2682 00000100:00080000:2.0:1775766060.741850:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766060.741857:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766060.741890:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1c00 x1862022967184512/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766060.741914:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766060.741916:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766060.741918:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766060.741921:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2682 00000100:00080000:2.0:1775766060.741923:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766060.741925:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766060.741934:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766060.741942:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3b80 x1862022967184640/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766060.741949:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967184512:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775766060.741952:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766060.741957:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766060.741959:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766060.741962:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2682 00000100:00080000:2.0:1775766060.741964:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766060.741966:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766060.741970:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775766060.742010:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766060.742012:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1880 x1862022967184768/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766060.742021:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766060.742023:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766060.742025:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766060.742028:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2682 00000100:00080000:2.0:1775766060.742031:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766060.742033:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766060.742036:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766060.742044:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2a00 x1862022967184896/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766060.742062:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766060/real 1775766060] req@ffff8f99703d1c00 x1862022967184512/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766060.742100:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1c00 x1862022967184512/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766060.742108:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766060.742111:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766060.742114:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967184512:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766060.742132:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967184640:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766060.742149:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766060.742157:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766060/real 1775766060] req@ffff8f99703d3b80 x1862022967184640/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766060.742165:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3b80 x1862022967184640/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766060.742171:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766060.742173:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766060.742175:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967184640:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766060.742179:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967184768:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766060.742191:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766060.742201:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766060/real 1775766060] req@ffff8f99703d1880 x1862022967184768/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766060.742213:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1880 x1862022967184768/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766060.742240:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766060.742244:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766060.742247:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967184768:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766060.742253:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967184896:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766060.742273:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766060.742285:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766060/real 1775766060] req@ffff8f99703d2a00 x1862022967184896/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766060.742295:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2a00 x1862022967184896/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766060.742302:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766060.742306:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766060.742309:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967184896:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766061.637854:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766061.637943:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766062.661814:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766062.661870:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766063.685876:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766063.686062:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766064.709911:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766064.710009:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766065.733899:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766065.734019:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766065.861904:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766065.861920:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766065.861927:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766065.861943:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2687 00000100:00080000:0.0:1775766065.861961:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766065.861971:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766065.862045:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967185024/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766065.862114:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766065.862122:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766065.862126:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766065.862137:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2687 00000100:00080000:0.0:1775766065.862147:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766065.862168:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766065.862187:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766065.862200:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967185024:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766065.862216:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967185152/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766065.862233:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766065.862239:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766065.862242:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766065.862249:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2687 00000100:00080000:0.0:1775766065.862269:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1775766065.862279:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775766065.862284:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766065.862293:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766065.862310:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967185280/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766065.862323:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766065.862328:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766065.862331:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766065.862337:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2687 00000100:00080000:0.0:1775766065.862341:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775766065.862342:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766065/real 1775766065] req@ffff8f9970a77480 x1862022967185024/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766065.862353:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766065.862373:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766065.862383:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967185024/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766065.862398:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967185408/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766065.862461:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766065.862468:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766065.862474:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967185024:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766065.862511:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967185152:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766065.862541:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766065.862557:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766065/real 1775766065] req@ffff8f9970a77800 x1862022967185152/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766065.862570:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967185152/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766065.862580:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766065.862584:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766065.862586:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967185152:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766065.862599:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967185280:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766065.862617:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766065.862625:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766065/real 1775766065] req@ffff8f9970a76a00 x1862022967185280/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766065.862633:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967185280/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766065.862638:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766065.862641:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766065.862643:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967185280:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766065.862661:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967185408:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766065.862674:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766065.862681:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766065/real 1775766065] req@ffff8f9970a75500 x1862022967185408/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766065.862689:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967185408/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766065.862695:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766065.862697:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766065.862699:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967185408:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766066.757822:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766066.758075:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766067.781822:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766067.781955:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766068.805796:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766068.805878:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766069.829843:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766069.829988:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766070.853812:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766070.853946:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766070.982063:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766070.982077:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766070.982084:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766070.982100:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2692 00000100:00080000:3.0:1775766070.982120:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766070.982131:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766070.982189:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf100 x1862022967185536/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766070.982233:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766070.982238:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766070.982240:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766070.982247:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2692 00000100:00080000:3.0:1775766070.982252:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766070.982257:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766070.982269:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766070.982290:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddc00 x1862022967185664/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766070.982306:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775766070.982309:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967185536:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775766070.982311:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766070.982316:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766070.982324:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2692 00000100:00080000:3.0:1775766070.982348:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766070.982355:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766070.982362:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766070.982396:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967185792/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766070.982411:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1775766070.982419:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775766070.982430:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766070.982434:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766070.982442:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2692 00000100:00080000:3.0:1775766070.982449:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766070.982454:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766070.982461:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766070.982477:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766070/real 1775766070] req@ffff8f9970edf100 x1862022967185536/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766070.982481:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967185920/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766070.982533:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf100 x1862022967185536/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766070.982570:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766070.982584:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766070.982589:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967185536:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766070.982653:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967185664:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766070.982702:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766070.982722:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766070/real 1775766070] req@ffff8f9970eddc00 x1862022967185664/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766070.982776:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddc00 x1862022967185664/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766070.982795:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766070.982809:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766070.982815:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967185664:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766070.982823:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967185792:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766070.982851:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766070.982866:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766070/real 1775766070] req@ffff8f9970edd500 x1862022967185792/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766070.982895:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967185792/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766070.982907:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766070.982911:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766070.982916:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967185792:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766070.982925:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967185920:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766070.982951:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766070.982965:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766070/real 1775766070] req@ffff8f9970edc700 x1862022967185920/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766070.983007:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967185920/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766070.983018:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766070.983023:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766070.983027:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967185920:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766071.877908:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766071.877982:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766072.901992:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766072.902017:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766073.925804:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766073.925981:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766074.949911:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766074.950029:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766075.973865:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766075.973966:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766076.101872:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766076.101883:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766076.101888:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766076.101898:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2697 00000100:00080000:1.0:1775766076.101909:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766076.101916:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766076.101954:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967186048/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766076.101982:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766076.101985:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766076.101992:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766076.101995:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2697 00000100:00080000:1.0:1775766076.101998:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766076.102001:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766076.102031:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967186048:192.168.202.146@tcp:250:kworker.0 00000008:00100000:1.0:1775766076.102062:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766076.102082:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967186176/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766076.102093:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775766076.102094:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766076.102099:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766076.102101:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766076.102106:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2697 00000100:00080000:1.0:1775766076.102113:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766076.102116:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766076.102123:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766076.102133:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766076/real 1775766076] req@ffff8f994f2e7b80 x1862022967186048/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766076.102149:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967186304/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766076.102159:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775766076.102162:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967186048/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766076.102165:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766076.102169:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766076.102178:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2697 00000100:00080000:1.0:1775766076.102183:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766076.102184:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766076.102187:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00080000:0.0:1775766076.102193:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000008:00100000:1.0:1775766076.102195:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766076.102197:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967186048:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766076.102216:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967186432/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766076.102224:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967186176:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775766076.102251:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766076.102263:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766076/real 1775766076] req@ffff8f994f2e4e00 x1862022967186176/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766076.102273:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967186176/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766076.102280:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766076.102282:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766076.102285:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967186176:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775766076.102289:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967186304:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766076.102301:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766076.102307:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766076/real 1775766076] req@ffff8f994f2e6300 x1862022967186304/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766076.102321:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967186304/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766076.102404:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766076.102407:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766076.102411:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967186304:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766076.102427:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967186432:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766076.102446:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766076.102455:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766076/real 1775766076] req@ffff8f994f2e4700 x1862022967186432/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766076.102464:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967186432/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766076.102470:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766076.102472:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766076.102474:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967186432:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766076.997934:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766076.998084:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766078.021890:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766078.022010:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766079.045947:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766079.046092:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766080.069959:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766080.069969:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766081.093879:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766081.093924:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766081.221875:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766081.221883:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766081.221889:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766081.221902:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2702 00000100:00080000:0.0:1775766081.221912:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766081.221917:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766081.221951:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967186560/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766081.221977:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775766081.222013:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967186560:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775766081.222018:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766081.222022:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766081.222026:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2702 00000100:00080000:0.0:1775766081.222029:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766081.222032:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766081.222040:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766081.222053:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967186688/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766081.222068:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766081.222072:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766081.222074:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1775766081.222078:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775766081.222087:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2702 00000100:00080000:0.0:1775766081.222092:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766081.222098:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766081.222107:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766081.222128:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967186816/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766081.222135:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766081/real 1775766081] req@ffff8f9970a77100 x1862022967186560/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766081.222144:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766081.222150:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766081.222152:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766081.222156:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2702 00000100:00100000:1.0:1775766081.222160:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967186560/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766081.222161:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766081.222168:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00080000:1.0:1775766081.222175:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766081.222180:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766081.222189:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967186560:192.168.202.146@tcp:250:kworker.0 00000008:00100000:0.0:1775766081.222191:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766081.222207:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967186944/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766081.222225:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967186688:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766081.222253:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766081.222292:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766081/real 1775766081] req@ffff8f9970a76300 x1862022967186688/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766081.222306:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967186688/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766081.222312:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766081.222315:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766081.222317:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967186688:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766081.222328:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967186816:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766081.222346:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766081.222373:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766081/real 1775766081] req@ffff8f9970a75180 x1862022967186816/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766081.222381:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967186816/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766081.222387:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766081.222390:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766081.222393:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967186816:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766081.222396:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967186944:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766081.222409:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766081.222417:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766081/real 1775766081] req@ffff8f9970a75500 x1862022967186944/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766081.222425:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967186944/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766081.222430:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766081.222434:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766081.222436:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967186944:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766082.117856:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766082.117897:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766083.141861:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766083.141928:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766084.165803:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766084.165892:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766085.189834:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766085.189946:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766086.213840:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766086.213929:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766086.341850:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766086.341859:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766086.341867:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766086.341875:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2707 00000100:00080000:2.0:1775766086.341885:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766086.341891:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766086.341932:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2a00 x1862022967187072/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766086.341953:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766086.341955:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766086.341957:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766086.341960:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2707 00000100:00080000:2.0:1775766086.341963:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766086.341966:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766086.341974:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766086.341982:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967187072:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766086.341985:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1880 x1862022967187200/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766086.341996:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766086.342031:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766086.342034:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766086.342037:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2707 00000100:00080000:2.0:1775766086.342039:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766086.342043:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:1.0:1775766086.342048:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:2.0:1775766086.342051:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766086.342062:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3b80 x1862022967187328/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766086.342069:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766086.342071:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766086.342073:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766086.342075:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2707 00000100:00080000:2.0:1775766086.342079:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766086.342081:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766086.342083:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766086.342092:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1c00 x1862022967187456/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766086.342098:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766086/real 1775766086] req@ffff8f99703d2a00 x1862022967187072/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766086.342130:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2a00 x1862022967187072/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766086.342139:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766086.342142:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766086.342145:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967187072:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766086.342163:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967187200:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766086.342179:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766086.342188:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766086/real 1775766086] req@ffff8f99703d1880 x1862022967187200/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766086.342198:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1880 x1862022967187200/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766086.342207:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766086.342211:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766086.342214:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967187200:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766086.342220:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967187328:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766086.342254:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766086.342279:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766086/real 1775766086] req@ffff8f99703d3b80 x1862022967187328/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766086.342292:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3b80 x1862022967187328/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766086.342315:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766086.342317:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766086.342320:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967187328:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766086.342324:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967187456:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766086.342342:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766086.342353:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766086/real 1775766086] req@ffff8f99703d1c00 x1862022967187456/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766086.342366:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1c00 x1862022967187456/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766086.342375:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766086.342378:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766086.342382:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967187456:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775766087.238017:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766087.238054:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775766088.261826:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766088.261977:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766089.285848:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766089.285922:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766090.309851:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766090.309925:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766091.333870:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766091.334019:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766091.461817:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766091.461829:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766091.461901:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766091.461915:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2713 00000100:00080000:2.0:1775766091.461930:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766091.461936:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766091.461983:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2300 x1862022967187584/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766091.462013:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766091.462016:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766091.462018:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766091.462021:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2713 00000100:00080000:2.0:1775766091.462024:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766091.462027:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766091.462065:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766091.462080:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3480 x1862022967187712/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766091.462098:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766091.462104:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766091.462107:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775766091.462128:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967187584:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775766091.462138:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2713 00000100:00080000:2.0:1775766091.462151:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766091.462161:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766091.462170:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766091.462243:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3100 x1862022967187840/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766091.462269:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766091.462273:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766091.462274:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766091.462278:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2713 00000100:00080000:2.0:1775766091.462283:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1775766091.462293:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775766091.462296:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766091.462303:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766091.462323:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1180 x1862022967187968/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766091.462356:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766091/real 1775766091] req@ffff8f99703d2300 x1862022967187584/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766091.462376:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2300 x1862022967187584/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766091.462421:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766091.462426:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766091.462430:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967187584:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766091.462456:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967187712:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766091.462483:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766091.462495:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766091/real 1775766091] req@ffff8f99703d3480 x1862022967187712/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766091.462507:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3480 x1862022967187712/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766091.462515:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766091.462518:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766091.462521:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967187712:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766091.462527:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967187840:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766091.462543:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766091.462568:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766091/real 1775766091] req@ffff8f99703d3100 x1862022967187840/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766091.462583:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3100 x1862022967187840/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766091.462593:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766091.462597:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766091.462600:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967187840:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766091.462645:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967187968:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766091.462673:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766091.462687:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766091/real 1775766091] req@ffff8f99703d1180 x1862022967187968/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766091.462702:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1180 x1862022967187968/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766091.462712:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766091.462716:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766091.462720:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967187968:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766092.357882:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766092.357951:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766093.381880:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766093.381881:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766094.405985:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766094.405988:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775766095.429844:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766095.429931:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766096.453894:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766096.453997:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766096.581928:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766096.581941:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766096.581948:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766096.581962:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2718 00000100:00080000:3.0:1775766096.581978:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766096.581986:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766096.582049:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967188096/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766096.582079:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766096.582084:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766096.582087:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766096.582093:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2718 00000100:00080000:3.0:1775766096.582096:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766096.582101:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766096.582113:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766096.582123:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967188096:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766096.582160:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967188224/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766096.582200:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766096.582207:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766096.582210:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766096.582216:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2718 00000100:00080000:3.0:1775766096.582221:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766096.582225:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766096.582233:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766096.582251:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967188352/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766096.582260:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766096.582263:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766096.582265:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766096.582270:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2718 00000800:00000100:1.0:1775766096.582281:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775766096.582284:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766096.582292:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766096.582299:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766096.582317:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967188480/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766096.582375:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766096/real 1775766096] req@ffff8f9970edce00 x1862022967188096/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766096.582402:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967188096/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766096.582418:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766096.582425:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766096.582432:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967188096:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766096.582523:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967188224:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766096.582553:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766096.582570:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766096/real 1775766096] req@ffff8f9970eded80 x1862022967188224/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766096.582586:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967188224/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766096.582596:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766096.582601:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766096.582605:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967188224:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766096.582612:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967188352:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766096.582631:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766096.582642:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766096/real 1775766096] req@ffff8f9970ede680 x1862022967188352/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766096.582656:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967188352/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766096.582666:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766096.582669:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766096.582673:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967188352:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766096.582680:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967188480:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766096.582697:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766096.582707:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766096/real 1775766096] req@ffff8f9970edc000 x1862022967188480/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766096.582792:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967188480/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766096.582803:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766096.582807:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766096.582811:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967188480:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766097.477893:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766097.478001:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766098.501920:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766098.501981:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766099.525863:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766099.525935:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766100.549888:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766100.550146:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766101.573861:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766101.574010:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00080000:2.0:1775766101.701927:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766101.701943:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766101.701950:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766101.701966:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2723 00000100:00080000:2.0:1775766101.701985:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766101.701996:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766101.702145:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1500 x1862022967188608/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766101.702186:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766101.702191:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766101.702194:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766101.702200:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2723 00000100:00080000:2.0:1775766101.702207:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766101.702214:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766101.702230:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766101.702252:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967188608:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766101.702252:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0700 x1862022967188736/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766101.702290:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766101.702294:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766101.702297:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766101.702305:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2723 00000100:00080000:2.0:1775766101.702308:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766101.702316:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766101.702322:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766101.702340:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0e00 x1862022967188864/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766101.702353:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766101.702357:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766101.702359:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1775766101.702364:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775766101.702391:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2723 00000100:00080000:2.0:1775766101.702399:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766101.702405:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766101.702413:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766101.702423:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766101/real 1775766101] req@ffff8f99703d1500 x1862022967188608/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766101.702435:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2d80 x1862022967188992/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766101.702461:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1500 x1862022967188608/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766101.702476:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766101.702480:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766101.702485:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967188608:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766101.702518:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967188736:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766101.702563:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766101.702583:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766101/real 1775766101] req@ffff8f99703d0700 x1862022967188736/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766101.702674:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0700 x1862022967188736/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766101.702683:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766101.702686:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766101.702689:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967188736:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766101.702696:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967188864:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766101.702720:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766101.702731:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766101/real 1775766101] req@ffff8f99703d0e00 x1862022967188864/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766101.702783:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0e00 x1862022967188864/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766101.702791:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766101.702794:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766101.702797:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967188864:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766101.702804:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967188992:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766101.702821:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766101.702831:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766101/real 1775766101] req@ffff8f99703d2d80 x1862022967188992/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766101.702841:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2d80 x1862022967188992/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766101.702875:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766101.702881:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766101.702885:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967188992:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766102.597880:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766102.598035:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766103.621847:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766103.621971:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766104.645866:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766104.645954:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766105.669838:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766105.669951:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766106.693867:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766106.693955:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766106.821882:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766106.821891:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766106.821895:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766106.821903:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2728 00000100:00080000:3.0:1775766106.821913:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766106.821919:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766106.821974:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967189120/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766106.822024:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766106.822028:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766106.822029:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766106.822045:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2728 00000100:00080000:3.0:1775766106.822049:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766106.822052:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766106.822061:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967189120:192.168.202.146@tcp:250:kworker.0 00000008:00100000:3.0:1775766106.822061:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766106.822073:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967189248/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766106.822092:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766106.822095:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766106.822097:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766106.822099:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2728 00000100:00080000:3.0:1775766106.822102:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766106.822104:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766106.822107:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766106.822115:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967189376/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766106.822121:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766106.822124:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766106.822125:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766106.822127:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2728 00000800:00000100:1.0:1775766106.822141:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775766106.822174:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766106.822177:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766106.822181:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766106.822188:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967189504/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766106.822206:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766106/real 1775766106] req@ffff8f9970ede300 x1862022967189120/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766106.822227:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967189120/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766106.822243:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766106.822249:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766106.822253:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967189120:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766106.822280:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967189248:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766106.822299:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766106.822325:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766106/real 1775766106] req@ffff8f9970edf480 x1862022967189248/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766106.822349:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967189248/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766106.822388:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766106.822393:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766106.822396:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967189248:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766106.822403:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967189376:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766106.822425:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766106.822436:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766106/real 1775766106] req@ffff8f9970edd880 x1862022967189376/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766106.822449:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967189376/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766106.822457:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766106.822460:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766106.822463:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967189376:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766106.822470:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967189504:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766106.822489:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766106.822500:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766106/real 1775766106] req@ffff8f9970edf800 x1862022967189504/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766106.822512:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967189504/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766106.822520:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766106.822523:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766106.822542:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967189504:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766107.717939:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766107.717982:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766108.741909:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766108.741941:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775766109.765851:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766109.765937:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766110.789847:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766110.789928:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766111.813843:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766111.814060:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766111.941938:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766111.941950:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766111.941955:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766111.941966:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2733 00000100:00080000:2.0:1775766111.941978:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766111.942012:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766111.942056:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3800 x1862022967189632/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766111.942086:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766111.942090:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1775766111.942143:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967189632:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775766111.942165:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766111.942173:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2733 00000100:00080000:2.0:1775766111.942179:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766111.942183:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766111.942196:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766111.942221:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2680 x1862022967189760/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766111.942240:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766111.942245:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766111.942248:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766111.942265:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2733 00000100:00080000:2.0:1775766111.942272:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766111.942278:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766111.942288:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766111.942290:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967189760:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766111.942318:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0000 x1862022967189888/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766111.942338:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766111.942344:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766111.942347:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775766111.942353:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967189888:192.168.202.146@tcp:8:kworker.0 00000100:00080000:2.0:1775766111.942361:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2733 00000100:00080000:2.0:1775766111.942370:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766111.942376:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766111.942385:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766111.942434:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0380 x1862022967190016/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766111.942509:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967190016:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766112.837822:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766112.837883:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766113.861837:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766113.861900:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766114.885837:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766114.885953:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775766115.014073:0:2366:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:3.0:1775766115.014120:0:2366:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775766115.014135:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:3.0:1775766115.014171:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:3.0:1775766115.014179:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:3.0:1775766115.014185:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:1.0:1775766115.014253:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766111/real 1775766115] req@ffff8f99703d0380 x1862022967190016/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766115.014303:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0380 x1862022967190016/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766115.014313:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766115.014318:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766115.014322:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967190016:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766115.014330:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766111/real 1775766115] req@ffff8f99703d0000 x1862022967189888/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766115.014345:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0000 x1862022967189888/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766115.014421:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766115.014430:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766115.014448:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967189888:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766115.014463:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766111/real 1775766115] req@ffff8f99703d2680 x1862022967189760/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766115.014491:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2680 x1862022967189760/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766115.014507:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766115.014515:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766115.014522:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967189760:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766115.014535:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766111/real 1775766115] req@ffff8f99703d3800 x1862022967189632/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766115.014572:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3800 x1862022967189632/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766115.014600:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766115.014604:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766115.014608:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967189632:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766115.910044:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766115.910075:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766116.933865:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766116.933967:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766117.061947:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766117.061956:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766117.061959:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766117.061970:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2738 00000100:00080000:3.0:1775766117.061992:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766117.061999:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766117.062032:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967190144/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766117.062060:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766117.062063:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766117.062065:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766117.062068:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2738 00000100:00080000:3.0:1775766117.062071:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766117.062075:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766117.062083:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766117.062096:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967190144:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766117.062102:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967190272/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766117.062118:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766117.062122:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766117.062126:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766117.062131:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2738 00000100:00080000:3.0:1775766117.062136:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766117.062142:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766117.062148:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775766117.062164:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766117.062172:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967190400/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766117.062184:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766117.062188:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766117.062190:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766117.062196:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2738 00000100:00100000:2.0:1775766117.062207:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766117/real 1775766117] req@ffff8f9970edea00 x1862022967190144/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766117.062210:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766117.062219:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766117.062226:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766117.062232:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967190144/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766117.062240:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1775766117.062245:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967190528/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766117.062248:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766117.062256:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967190144:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766117.062295:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967190272:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766117.062320:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766117.062330:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766117/real 1775766117] req@ffff8f9970edca80 x1862022967190272/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766117.062352:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967190272/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766117.062361:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766117.062419:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766117.062424:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967190272:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766117.062431:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967190400:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766117.062458:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766117.062469:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766117/real 1775766117] req@ffff8f9970edc380 x1862022967190400/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766117.062481:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967190400/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766117.062489:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766117.062492:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766117.062495:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967190400:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766117.062500:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967190528:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766117.062522:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766117.062539:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766117/real 1775766117] req@ffff8f9970edfb80 x1862022967190528/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766117.062551:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967190528/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766117.062560:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766117.062563:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766117.062566:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967190528:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766117.957821:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766117.957912:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766118.981953:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766118.982013:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766120.005833:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766120.005873:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766121.029798:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766121.029878:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766122.053928:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766122.053998:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766122.181925:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766122.181944:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766122.181952:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766122.181971:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2743 00000100:00080000:3.0:1775766122.181991:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766122.182005:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766122.182078:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd180 x1862022967190656/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766122.182122:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766122.182127:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766122.182130:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766122.182136:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2743 00000100:00080000:3.0:1775766122.182141:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766122.182147:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766122.182162:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766122.182178:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddf80 x1862022967190784/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766122.182194:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775766122.182207:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967190656:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775766122.182207:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766122.182212:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766122.182231:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2743 00000100:00080000:3.0:1775766122.182235:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766122.182240:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766122.182245:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766122.182264:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967190912/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766122.182278:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766122.182282:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766122.182285:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766122.182291:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2743 00000100:00080000:3.0:1775766122.182306:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766122.182313:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766122.182319:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775766122.182328:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766122.182335:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967191040/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766122.182414:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766122/real 1775766122] req@ffff8f9970edd180 x1862022967190656/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766122.182441:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd180 x1862022967190656/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766122.182457:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766122.182463:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766122.182468:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967190656:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766122.182498:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967190784:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766122.182529:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766122.182550:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766122/real 1775766122] req@ffff8f9970eddf80 x1862022967190784/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766122.182574:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddf80 x1862022967190784/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766122.182592:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766122.182599:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766122.182606:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967190784:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766122.182621:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967190912:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766122.182713:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766122.182730:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766122/real 1775766122] req@ffff8f9970edfb80 x1862022967190912/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766122.182790:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967190912/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766122.182806:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766122.182813:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766122.182819:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967190912:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766122.182833:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967191040:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766122.182874:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766122.182929:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766122/real 1775766122] req@ffff8f9970edc380 x1862022967191040/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766122.182946:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967191040/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766122.182957:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766122.182961:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766122.182965:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967191040:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766123.077989:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766123.078106:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766124.101823:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766124.101930:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766125.125802:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766125.126061:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766126.149882:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766126.150006:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766127.173806:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766127.173887:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766127.301988:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766127.301999:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766127.302004:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766127.302017:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2748 00000100:00080000:0.0:1775766127.302029:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766127.302036:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766127.302083:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967191168/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766127.302113:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766127.302116:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766127.302118:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766127.302122:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2748 00000100:00080000:0.0:1775766127.302125:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766127.302128:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766127.302175:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967191168:192.168.202.146@tcp:250:kworker.0 00000008:00100000:0.0:1775766127.302261:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775766127.302285:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766127.302292:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967191296/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766127.302313:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766127.302318:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766127.302321:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766127.302329:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2748 00000100:00100000:2.0:1775766127.302330:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766127/real 1775766127] req@ffff8f9970a76a00 x1862022967191168/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766127.302338:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766127.302349:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766127.302351:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967191168/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1775766127.302357:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1775766127.302360:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766127.302365:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766127.302369:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967191168:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766127.302371:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967191424/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766127.302381:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766127.302384:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766127.302385:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766127.302388:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2748 00000100:00080000:0.0:1775766127.302391:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775766127.302392:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967191296:192.168.202.146@tcp:38:kworker.0 00000100:00080000:0.0:1775766127.302398:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766127.302402:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766127.302413:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967191552/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766127.302419:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766127.302434:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766127/real 1775766127] req@ffff8f9970a77800 x1862022967191296/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766127.302444:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967191296/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766127.302450:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766127.302453:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766127.302456:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967191296:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766127.302460:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967191424:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766127.302471:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766127.302477:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766127/real 1775766127] req@ffff8f9970a77480 x1862022967191424/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766127.302502:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967191424/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766127.302507:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766127.302509:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766127.302512:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967191424:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766127.302520:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967191552:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766127.302532:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766127.302538:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766127/real 1775766127] req@ffff8f9970a74380 x1862022967191552/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766127.302545:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967191552/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766127.302551:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766127.302552:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766127.302554:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967191552:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766128.197882:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766128.197937:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766129.221899:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766129.222114:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766130.245923:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766130.245924:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775766131.269885:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766131.269986:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766132.293897:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766132.293913:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766132.421900:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766132.421908:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766132.421912:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766132.421921:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2754 00000100:00080000:3.0:1775766132.421930:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766132.421936:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766132.421968:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967191680/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766132.421992:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766132.421995:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766132.421996:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766132.421999:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2754 00000100:00080000:3.0:1775766132.422002:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766132.422004:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766132.422011:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766132.422020:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967191808/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766132.422028:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766132.422031:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766132.422032:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766132.422035:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2754 00000100:00080000:3.0:1775766132.422036:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766132.422038:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766132.422041:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766132.422050:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967191936/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766132.422058:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766132.422060:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766132.422061:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766132.422068:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2754 00000100:00100000:2.0:1775766132.422070:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967191680:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775766132.422071:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766132.422078:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766132.422088:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766132.422095:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967192064/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766132.422164:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766132.422219:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766132/real 1775766132] req@ffff8f9970edca80 x1862022967191680/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766132.422429:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967191680/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766132.422438:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766132.422442:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766132.422445:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967191680:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766132.422454:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967191808:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766132.422502:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766132.422520:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766132/real 1775766132] req@ffff8f9970edea00 x1862022967191808/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766132.422528:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967191808/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766132.422534:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766132.422535:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766132.422537:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967191808:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766132.422557:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967191936:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766132.422570:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766132.422576:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766132/real 1775766132] req@ffff8f9970edf800 x1862022967191936/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766132.422583:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967191936/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766132.422612:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766132.422615:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766132.422617:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967191936:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766132.422621:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967192064:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766132.422633:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766132.422639:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766132/real 1775766132] req@ffff8f9970edd880 x1862022967192064/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766132.422646:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967192064/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766132.422651:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766132.422654:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766132.422655:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967192064:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766133.317860:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766133.317924:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766134.341831:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766134.341922:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766135.365847:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766135.365951:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766136.389856:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766136.389935:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766137.413899:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766137.413951:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766137.542735:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766137.542827:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766137.542833:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766137.542844:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2759 00000100:00080000:3.0:1775766137.542856:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766137.542863:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766137.542900:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967192192/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766137.542923:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766137.542926:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766137.542927:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766137.542931:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2759 00000100:00080000:3.0:1775766137.542934:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766137.542937:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766137.542946:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766137.542956:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967192320/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766137.542960:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967192192:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775766137.542971:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766137.542974:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766137.542975:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766137.542979:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2759 00000100:00080000:3.0:1775766137.542980:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766137.542982:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766137.542985:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766137.542994:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967192448/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766137.543005:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766137.543008:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766137.543010:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766137.543012:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2759 00000800:00000100:2.0:1775766137.543035:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766137.543068:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766137/real 1775766137] req@ffff8f9970edf480 x1862022967192192/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766137.543085:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967192192/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766137.543108:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766137.543113:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766137.543118:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1775766137.543129:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766137.543133:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766137.543137:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967192192:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766137.543145:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967192576/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766137.543159:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967192320:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766137.543179:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766137.543188:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766137/real 1775766137] req@ffff8f9970ede300 x1862022967192320/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766137.543196:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967192320/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766137.543202:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766137.543204:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766137.543206:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967192320:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766137.543210:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967192448:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766137.543221:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766137.543227:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766137/real 1775766137] req@ffff8f9970edc000 x1862022967192448/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766137.543234:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967192448/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766137.543240:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766137.543241:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766137.543243:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967192448:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766137.543259:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967192576:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766137.543272:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766137.543279:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766137/real 1775766137] req@ffff8f9970ede680 x1862022967192576/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766137.543287:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967192576/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766137.543292:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766137.543294:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766137.543297:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967192576:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766138.437818:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766138.437880:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766139.461854:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766139.461926:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766140.485817:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766140.485964:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766141.509870:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766141.509922:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766142.533814:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766142.533892:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766142.661850:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766142.661862:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766142.661867:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766142.661880:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2764 00000100:00080000:1.0:1775766142.661892:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766142.661900:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766142.661937:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967192704/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766142.661964:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766142.661968:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766142.661971:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766142.661975:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2764 00000100:00080000:1.0:1775766142.661979:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775766142.662006:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967192704:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775766142.662014:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766142.662024:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766142.662039:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967192832/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766142.662051:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766142.662054:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766142.662057:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766142.662061:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2764 00000100:00080000:1.0:1775766142.662066:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766142.662071:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:2.0:1775766142.662077:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:1.0:1775766142.662078:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766142.662093:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967192960/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766142.662104:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766142.662107:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766142.662117:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766142.662121:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2764 00000100:00080000:1.0:1775766142.662125:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766142.662129:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766142.662134:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766142.662140:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766142/real 1775766142] req@ffff8f994f2e6680 x1862022967192704/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766142.662153:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967193088/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766142.662164:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967192704/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766142.662177:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766142.662183:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766142.662189:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967192704:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766142.662236:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967192832:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766142.662262:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766142.662276:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766142/real 1775766142] req@ffff8f994f2e5f80 x1862022967192832/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766142.662290:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967192832/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766142.662299:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766142.662303:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766142.662306:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967192832:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766142.662312:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967192960:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766142.662333:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766142.662346:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766142/real 1775766142] req@ffff8f994f2e6a00 x1862022967192960/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766142.662358:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967192960/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766142.662365:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766142.662368:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766142.662371:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967192960:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766142.662377:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967193088:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766142.662392:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766142.662401:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766142/real 1775766142] req@ffff8f994f2e4000 x1862022967193088/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766142.662428:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967193088/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766142.662436:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766142.662439:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766142.662442:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967193088:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766143.557826:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766143.557886:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766144.581893:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766144.581917:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766145.605804:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766145.605876:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766146.629792:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766146.629865:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766147.653809:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766147.653884:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766147.781838:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766147.781847:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766147.781851:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766147.781860:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2769 00000100:00080000:0.0:1775766147.781869:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766147.781874:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766147.781920:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967193216/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766147.781944:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766147.781947:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766147.781948:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766147.781952:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2769 00000100:00080000:0.0:1775766147.781954:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766147.781957:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766147.781971:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766147.781980:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967193344/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766147.781985:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967193216:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775766147.781990:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766147.781996:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766147.781997:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766147.782000:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2769 00000100:00080000:0.0:1775766147.782002:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766147.782004:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766147.782008:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766147.782016:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967193472/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766147.782022:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766147.782024:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766147.782026:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766147.782028:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2769 00000100:00080000:0.0:1775766147.782031:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766147.782033:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766147.782036:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766147.782044:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967193600/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766147.782056:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766147.782098:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766147/real 1775766147] req@ffff8f9970a75c00 x1862022967193216/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766147.782114:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967193216/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766147.782122:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766147.782125:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766147.782128:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967193216:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766147.782146:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967193344:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766147.782162:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766147.782170:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766147/real 1775766147] req@ffff8f9970a74700 x1862022967193344/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766147.782214:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967193344/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766147.782220:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766147.782222:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766147.782224:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967193344:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766147.782228:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967193472:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766147.782241:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766147.782247:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766147/real 1775766147] req@ffff8f9970a76d80 x1862022967193472/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766147.782254:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967193472/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766147.782259:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766147.782261:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766147.782263:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967193472:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766147.782266:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967193600:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766147.782275:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766147.782280:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766147/real 1775766147] req@ffff8f9970a74380 x1862022967193600/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766147.782287:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967193600/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766147.782307:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766147.782309:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766147.782311:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967193600:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766148.677875:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766148.677976:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766149.701811:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766149.701864:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766150.725850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766150.725926:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766151.749868:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766151.749873:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766152.773850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766152.773909:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766152.901865:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766152.901873:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766152.901877:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766152.901886:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2774 00000100:00080000:0.0:1775766152.901897:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766152.901904:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766152.901937:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967193728/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766152.901960:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766152.901963:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766152.901964:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766152.901968:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2774 00000100:00080000:0.0:1775766152.901972:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766152.901977:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766152.901988:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766152.901989:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967193728:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766152.902010:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967193856/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766152.902024:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766152.902029:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766152.902031:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766152.902038:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2774 00000100:00080000:0.0:1775766152.902043:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766152.902047:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766152.902052:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775766152.902058:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766152.902066:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967193984/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766152.902078:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766152.902082:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766152.902084:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766152.902088:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2774 00000100:00080000:0.0:1775766152.902093:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775766152.902097:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766152/real 1775766152] req@ffff8f9970a77480 x1862022967193728/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766152.902100:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766152.902111:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766152.902118:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967193728/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766152.902128:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967194112/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766152.902128:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766152.902134:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766152.902141:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967193728:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766152.902164:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967193856:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766152.902182:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766152.902191:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766152/real 1775766152] req@ffff8f9970a77800 x1862022967193856/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766152.902200:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967193856/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766152.902218:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766152.902221:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766152.902259:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967193856:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766152.902263:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967193984:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766152.902278:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766152.902287:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766152/real 1775766152] req@ffff8f9970a76a00 x1862022967193984/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766152.902296:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967193984/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766152.902301:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766152.902303:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766152.902306:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967193984:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766152.902310:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967194112:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766152.902322:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766152.902328:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766152/real 1775766152] req@ffff8f9970a75500 x1862022967194112/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766152.902335:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967194112/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766152.902341:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766152.902343:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766152.902345:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967194112:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766153.797854:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766153.797924:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766154.821820:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766154.821888:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766155.845879:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766155.845899:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766156.869839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766156.869921:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766157.893838:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766157.893900:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766158.021846:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766158.021858:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766158.021862:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766158.021873:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2779 00000100:00080000:2.0:1775766158.021885:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766158.021893:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766158.021930:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3800 x1862022967194240/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766158.021953:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766158.021957:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766158.021960:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766158.021964:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2779 00000100:00080000:2.0:1775766158.021968:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766158.021972:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766158.021985:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967194240:192.168.202.146@tcp:250:kworker.0 00000008:00100000:2.0:1775766158.022005:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766158.022018:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2680 x1862022967194368/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766158.022030:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766158.022033:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766158.022035:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766158.022038:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2779 00000100:00080000:2.0:1775766158.022041:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766158.022044:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766158.022049:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775766158.022056:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766158.022058:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0000 x1862022967194496/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766158.022067:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766158.022069:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766158.022071:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766158.022075:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2779 00000100:00080000:2.0:1775766158.022077:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766158.022080:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766158.022085:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766158.022093:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0380 x1862022967194624/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766158.022103:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766158/real 1775766158] req@ffff8f99703d3800 x1862022967194240/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766158.022124:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3800 x1862022967194240/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766158.022136:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766158.022140:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766158.022143:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967194240:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766158.022162:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967194368:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766158.022181:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766158.022191:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766158/real 1775766158] req@ffff8f99703d2680 x1862022967194368/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766158.022199:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2680 x1862022967194368/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766158.022204:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766158.022206:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766158.022209:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967194368:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766158.022212:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967194496:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766158.022232:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766158.022253:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766158/real 1775766158] req@ffff8f99703d0000 x1862022967194496/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766158.022261:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0000 x1862022967194496/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766158.022267:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766158.022269:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766158.022271:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967194496:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766158.022275:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967194624:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766158.022287:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766158.022293:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766158/real 1775766158] req@ffff8f99703d0380 x1862022967194624/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766158.022300:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0380 x1862022967194624/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766158.022305:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766158.022307:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766158.022309:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967194624:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766158.917866:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766158.917949:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766159.941847:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766159.941956:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766160.965828:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766160.965889:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766161.989858:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766161.989932:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766163.013863:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766163.013944:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766163.141874:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766163.142032:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766163.142037:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766163.142058:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2784 00000100:00080000:2.0:1775766163.142071:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766163.142078:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766163.142129:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2d80 x1862022967194752/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766163.142153:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766163.142156:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766163.142158:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766163.142161:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2784 00000100:00080000:2.0:1775766163.142164:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766163.142167:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766163.142175:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766163.142189:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0e00 x1862022967194880/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766163.142201:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967194752:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775766163.142202:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766163.142211:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766163.142213:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766163.142218:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2784 00000100:00080000:2.0:1775766163.142222:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766163.142226:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766163.142232:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766163.142243:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0700 x1862022967195008/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766163.142266:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775766163.142275:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766163.142279:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766163.142281:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766163.142286:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2784 00000100:00080000:2.0:1775766163.142290:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766163.142295:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766163.142300:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766163.142314:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1500 x1862022967195136/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766163.142328:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766163/real 1775766163] req@ffff8f99703d2d80 x1862022967194752/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766163.142361:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2d80 x1862022967194752/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766163.142369:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766163.142372:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766163.142375:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967194752:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766163.142394:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967194880:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766163.142417:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766163.142431:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766163/real 1775766163] req@ffff8f99703d0e00 x1862022967194880/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766163.142444:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0e00 x1862022967194880/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766163.142453:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766163.142457:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766163.142460:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967194880:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766163.142466:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967195008:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766163.142483:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766163.142490:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766163/real 1775766163] req@ffff8f99703d0700 x1862022967195008/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766163.142530:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0700 x1862022967195008/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766163.142564:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766163.142566:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766163.142568:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967195008:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766163.142573:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967195136:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766163.142588:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766163.142595:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766163/real 1775766163] req@ffff8f99703d1500 x1862022967195136/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766163.142602:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1500 x1862022967195136/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766163.142608:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766163.142610:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766163.142612:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967195136:192.168.202.146@tcp:8:kworker.0 00010000:00010000:0.0:1775766164.037899:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766164.037920:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:3.0:1775766165.061812:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766165.061948:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766166.085859:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766166.085917:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766167.109813:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766167.109909:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766168.133840:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766168.133977:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766168.261847:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766168.261857:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766168.261861:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766168.261869:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2789 00000100:00080000:0.0:1775766168.261880:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766168.261887:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766168.261922:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967195264/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766168.261953:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766168.261956:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766168.261958:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766168.261962:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2789 00000100:00080000:0.0:1775766168.261964:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766168.261975:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766168.261984:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766168.261994:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967195392/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766168.262010:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967195264:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775766168.262044:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766168.262047:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766168.262049:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766168.262053:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2789 00000100:00080000:0.0:1775766168.262063:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766168.262066:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766168.262071:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775766168.262076:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766168.262086:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967195520/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766168.262094:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766168.262098:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766168.262099:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766168.262103:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2789 00000100:00080000:0.0:1775766168.262106:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766168.262108:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766168.262112:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766168.262121:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967195648/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766168.262135:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766168/real 1775766168] req@ffff8f9970a75180 x1862022967195264/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766168.262171:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967195264/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766168.262207:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766168.262211:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766168.262214:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967195264:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766168.262254:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967195392:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766168.262279:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766168.262310:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766168/real 1775766168] req@ffff8f9970a76300 x1862022967195392/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766168.262327:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967195392/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766168.262341:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766168.262347:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766168.262352:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967195392:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766168.262363:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967195520:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766168.262398:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766168.262418:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766168/real 1775766168] req@ffff8f9970a77100 x1862022967195520/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766168.262440:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967195520/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766168.262519:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766168.262523:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766168.262528:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967195520:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766168.262553:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967195648:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766168.262573:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766168.262583:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766168/real 1775766168] req@ffff8f9970a75500 x1862022967195648/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766168.262592:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967195648/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766168.262609:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766168.262611:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766168.262614:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967195648:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766169.157812:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766169.157869:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766170.181834:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766170.181925:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766171.205827:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766171.205881:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766172.229866:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766172.229951:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766173.253938:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766173.253953:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00080000:0.0:1775766173.381931:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766173.381944:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766173.381950:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766173.381963:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2794 00000100:00080000:0.0:1775766173.382138:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766173.382150:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766173.382188:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967195776/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766173.382222:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766173.382225:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766173.382228:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766173.382232:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2794 00000100:00080000:0.0:1775766173.382234:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766173.382237:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766173.382246:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766173.382256:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967195904/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766173.382263:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766173.382265:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766173.382267:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766173.382270:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2794 00000100:00080000:0.0:1775766173.382273:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766173.382276:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766173.382280:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766173.382288:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967196032/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766173.382300:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766173.382305:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766173.382310:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775766173.382313:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967195776:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775766173.382320:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2794 00000100:00080000:0.0:1775766173.382328:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766173.382333:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766173.382354:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766173.382371:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967196160/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766173.382455:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766173.382524:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766173/real 1775766173] req@ffff8f9970a76a00 x1862022967195776/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766173.382552:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967195776/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766173.382567:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766173.382574:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766173.382580:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967195776:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766173.382663:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967195904:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766173.382688:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766173.382701:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766173/real 1775766173] req@ffff8f9970a77800 x1862022967195904/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766173.382712:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967195904/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766173.382721:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766173.382725:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766173.382730:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967195904:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766173.382795:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967196032:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766173.382822:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766173.382833:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766173/real 1775766173] req@ffff8f9970a77480 x1862022967196032/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766173.382843:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967196032/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766173.382852:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766173.382857:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766173.382861:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967196032:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766173.382869:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967196160:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766173.382893:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766173.382907:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766173/real 1775766173] req@ffff8f9970a74380 x1862022967196160/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766173.382945:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967196160/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766173.382952:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766173.382955:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766173.382958:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967196160:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766174.277829:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766174.277941:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766175.301862:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766175.301931:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766176.325850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766176.325944:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766177.349812:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766177.349918:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766178.373887:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766178.374015:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766178.501878:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766178.501889:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766178.501894:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766178.501905:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2800 00000100:00080000:3.0:1775766178.501918:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766178.501924:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766178.501971:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967196288/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766178.501997:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766178.502000:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766178.502001:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766178.502005:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2800 00000100:00080000:3.0:1775766178.502008:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766178.502012:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766178.502023:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766178.502036:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967196416/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766178.502047:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766178.502052:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766178.502054:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766178.502059:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2800 00000100:00080000:3.0:1775766178.502063:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766178.502068:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766178.502074:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766178.502081:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967196288:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766178.502091:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967196544/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766178.502099:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766178.502102:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766178.502104:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766178.502107:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2800 00000100:00080000:3.0:1775766178.502110:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766178.502112:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766178.502115:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766178.502125:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967196672/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766178.502202:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967196416:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766178.502239:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967196544:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766178.502258:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967196672:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766179.397808:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766179.397910:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766180.421918:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766180.421952:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766181.445806:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766181.445973:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775766181.574138:0:2368:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:3.0:1775766181.574177:0:2368:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775766181.574193:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:3.0:1775766181.574243:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:3.0:1775766181.574260:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:3.0:1775766181.574275:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:1.0:1775766181.574333:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766178/real 1775766181] req@ffff8f9970edc700 x1862022967196544/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766181.574379:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967196544/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766181.574397:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766181.574405:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766181.574451:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967196544:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766181.574464:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766178/real 1775766181] req@ffff8f9970edd500 x1862022967196672/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766181.574483:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967196672/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766181.574494:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766181.574499:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766181.574503:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967196672:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766181.574515:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766178/real 1775766181] req@ffff8f9970eded80 x1862022967196288/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766181.574533:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967196288/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766181.574542:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766181.574547:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766181.574551:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967196288:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766181.574562:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766178/real 1775766181] req@ffff8f9970edce00 x1862022967196416/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766181.574578:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967196416/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766181.574588:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766181.574592:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766181.574614:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967196416:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766182.469850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766182.469930:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766183.493836:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766183.493909:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766183.621810:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766183.621834:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766183.621838:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766183.621848:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2805 00000100:00080000:3.0:1775766183.621859:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766183.621879:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766183.621945:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddc00 x1862022967196800/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766183.622017:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766183.622021:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766183.622024:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766183.622029:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2805 00000100:00080000:3.0:1775766183.622034:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766183.622039:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766183.622049:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766183.622065:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf100 x1862022967196928/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766183.622084:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967196800:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775766183.622098:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766183.622103:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766183.622105:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766183.622111:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2805 00000100:00080000:3.0:1775766183.622115:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766183.622120:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766183.622126:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766183.622141:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967197056/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766183.622152:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766183.622159:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:1.0:1775766183.622160:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775766183.622162:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766183.622169:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2805 00000100:00080000:3.0:1775766183.622174:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766183.622178:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766183.622184:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766183.622197:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967197184/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766183.622234:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766183/real 1775766183] req@ffff8f9970eddc00 x1862022967196800/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766183.622258:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddc00 x1862022967196800/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766183.622271:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766183.622276:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766183.622281:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967196800:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766183.622345:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967196928:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766183.622376:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766183.622393:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766183/real 1775766183] req@ffff8f9970edf100 x1862022967196928/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766183.622410:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf100 x1862022967196928/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766183.622421:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766183.622425:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766183.622429:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967196928:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766183.622437:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967197056:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766183.622923:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766183.622963:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766183/real 1775766183] req@ffff8f9970edce00 x1862022967197056/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766183.622983:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967197056/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766183.622996:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766183.623001:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766183.623005:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967197056:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766183.623017:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967197184:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766183.623042:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766183.623054:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766183/real 1775766183] req@ffff8f9970eded80 x1862022967197184/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766183.623068:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967197184/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766183.623078:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766183.623082:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766183.623085:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967197184:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766184.517870:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766184.517920:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766185.541820:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766185.541902:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766186.565824:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766186.565934:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766187.589878:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766187.589885:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766188.613818:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766188.613913:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766188.741845:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766188.741854:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766188.741859:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766188.741867:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2810 00000100:00080000:2.0:1775766188.741884:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766188.741893:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766188.741930:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1180 x1862022967197312/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766188.741969:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766188.741974:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766188.741975:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766188.741980:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2810 00000100:00080000:2.0:1775766188.741985:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766188.741990:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766188.742001:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766188.742015:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3100 x1862022967197440/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766188.742025:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766188.742028:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766188.742030:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775766188.742058:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967197312:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775766188.742078:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2810 00000100:00080000:2.0:1775766188.742100:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766188.742105:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766188.742112:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766188.742127:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3480 x1862022967197568/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766188.742136:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766188.742139:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766188.742140:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766188.742143:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2810 00000100:00080000:2.0:1775766188.742146:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766188.742148:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766188.742152:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766188.742158:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766188.742164:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2300 x1862022967197696/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766188.742230:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766188/real 1775766188] req@ffff8f99703d1180 x1862022967197312/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766188.742268:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1180 x1862022967197312/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766188.742281:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766188.742286:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766188.742291:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967197312:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766188.742299:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967197440:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775766188.742322:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766188.742333:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766188/real 1775766188] req@ffff8f99703d3100 x1862022967197440/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766188.742342:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3100 x1862022967197440/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766188.742348:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766188.742350:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766188.742352:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967197440:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775766188.742374:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967197568:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766188.742387:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766188.742395:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766188/real 1775766188] req@ffff8f99703d3480 x1862022967197568/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766188.742404:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3480 x1862022967197568/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766188.742411:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766188.742415:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766188.742437:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967197568:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766188.742445:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967197696:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766188.742469:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766188.742497:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766188/real 1775766188] req@ffff8f99703d2300 x1862022967197696/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766188.742510:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2300 x1862022967197696/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766188.742521:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766188.742524:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766188.742527:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967197696:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766189.637887:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766189.638053:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766190.661816:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766190.661902:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766191.685872:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766191.685989:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766192.709839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766192.709922:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766193.733874:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766193.733990:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766193.861889:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766193.861898:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766193.861902:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766193.861910:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2815 00000100:00080000:3.0:1775766193.861920:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766193.861926:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766193.861959:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967197824/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766193.861985:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766193.861988:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766193.861989:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766193.861993:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2815 00000100:00080000:3.0:1775766193.861995:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766193.861997:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766193.862005:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766193.862013:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967197952/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766193.862019:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766193.862022:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766193.862023:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766193.862025:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2815 00000100:00080000:3.0:1775766193.862027:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766193.862029:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766193.862031:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766193.862038:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967198080/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766193.862043:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766193.862046:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766193.862047:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766193.862049:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2815 00000100:00080000:3.0:1775766193.862052:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766193.862054:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766193.862058:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766193.862067:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967198208/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766193.862110:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967197824:192.168.202.146@tcp:250:kworker.0 00000800:00000100:0.0:1775766193.862307:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766193.862374:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766193/real 1775766193] req@ffff8f9970edd500 x1862022967197824/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766193.862398:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967197824/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766193.862411:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766193.862415:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766193.862420:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967197824:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766193.862428:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967197952:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775766193.862480:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766193.862496:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766193/real 1775766193] req@ffff8f9970edc700 x1862022967197952/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766193.862510:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967197952/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766193.862520:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766193.862524:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766193.862708:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967197952:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775766193.862719:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967198080:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766193.862815:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766193.862839:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766193/real 1775766193] req@ffff8f9970ede680 x1862022967198080/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766193.862871:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967198080/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766193.862882:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766193.862886:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766193.862889:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967198080:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766193.862895:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967198208:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766193.862916:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766193.862928:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766193/real 1775766193] req@ffff8f9970edc000 x1862022967198208/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766193.862941:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967198208/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766193.862947:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766193.862949:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766193.862951:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967198208:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766194.757890:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766194.757940:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766195.781866:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766195.781956:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766196.805852:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766196.805922:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766197.829831:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766197.829889:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766198.853827:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766198.853896:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766198.981896:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766198.981940:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766198.981945:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766198.981955:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2820 00000100:00080000:0.0:1775766198.981966:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766198.981972:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766198.982011:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967198336/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766198.982041:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766198.982045:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766198.982047:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766198.982053:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2820 00000100:00080000:0.0:1775766198.982057:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766198.982061:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766198.982069:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967198336:192.168.202.146@tcp:250:kworker.0 00000008:00100000:0.0:1775766198.982072:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766198.982087:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967198464/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766198.982099:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766198.982103:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766198.982105:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766198.982109:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2820 00000100:00080000:0.0:1775766198.982126:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766198.982131:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766198.982137:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775766198.982138:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766198.982165:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967198592/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766198.982190:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766198.982194:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766198.982196:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775766198.982201:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766198/real 1775766198] req@ffff8f9970a76d80 x1862022967198336/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766198.982203:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2820 00000100:00080000:0.0:1775766198.982211:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766198.982218:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766198.982224:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766198.982225:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967198336/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766198.982236:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766198.982242:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766198.982243:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967198720/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766198.982246:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967198336:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766198.982274:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967198464:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766198.982302:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766198.982316:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766198/real 1775766198] req@ffff8f9970a74700 x1862022967198464/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766198.982339:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967198464/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766198.982345:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766198.982347:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766198.982350:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967198464:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766198.982472:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967198592:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766198.982533:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766198.982557:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766198/real 1775766198] req@ffff8f9970a75c00 x1862022967198592/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766198.982571:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967198592/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766198.982580:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766198.982583:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766198.982586:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967198592:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766198.982591:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967198720:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766198.982602:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766198.982608:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766198/real 1775766198] req@ffff8f9970a74380 x1862022967198720/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766198.982616:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967198720/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766198.982633:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766198.982636:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766198.982638:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967198720:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766199.877816:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766199.877887:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766200.901842:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766200.901988:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766201.925908:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766201.925973:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766202.949846:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766202.949963:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766203.973859:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766203.973932:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766204.102265:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766204.102278:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766204.102285:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766204.102301:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2825 00000100:00080000:2.0:1775766204.102317:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766204.102326:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766204.102378:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1c00 x1862022967198848/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766204.102419:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766204.102425:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766204.102428:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766204.102433:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2825 00000100:00080000:2.0:1775766204.102439:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766204.102445:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766204.102456:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766204.102476:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3b80 x1862022967198976/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766204.102488:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766204.102492:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766204.102494:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766204.102499:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2825 00000100:00100000:1.0:1775766204.102560:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967198848:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775766204.102571:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766204.102583:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766204.102597:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766204.102620:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1880 x1862022967199104/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766204.102640:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766204.102645:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766204.102648:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766204.102653:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2825 00000100:00080000:2.0:1775766204.102664:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766204.102670:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766204.102678:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766204.102699:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2a00 x1862022967199232/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766204.102721:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766204.102850:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766204/real 1775766204] req@ffff8f99703d1c00 x1862022967198848/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766204.102887:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1c00 x1862022967198848/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766204.102916:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766204.102924:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766204.102929:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967198848:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766204.102939:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967198976:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766204.102966:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766204.102979:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766204/real 1775766204] req@ffff8f99703d3b80 x1862022967198976/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766204.103000:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3b80 x1862022967198976/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766204.103007:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766204.103011:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766204.103014:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967198976:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766204.103058:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967199104:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766204.103076:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766204.103085:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766204/real 1775766204] req@ffff8f99703d1880 x1862022967199104/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766204.103158:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1880 x1862022967199104/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766204.103166:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766204.103168:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766204.103171:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967199104:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766204.103177:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967199232:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766204.103196:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766204.103205:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766204/real 1775766204] req@ffff8f99703d2a00 x1862022967199232/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766204.103214:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2a00 x1862022967199232/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766204.103220:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766204.103222:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766204.103225:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967199232:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766204.997896:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766204.998059:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766206.021887:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766206.022022:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766207.045800:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766207.045909:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766208.069837:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766208.069903:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766209.093941:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766209.093951:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766209.221887:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766209.221897:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766209.221903:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766209.221915:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2830 00000100:00080000:0.0:1775766209.221928:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766209.221935:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766209.221996:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967199360/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766209.222024:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766209.222028:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766209.222029:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766209.222033:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2830 00000100:00100000:1.0:1775766209.222088:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967199360:192.168.202.146@tcp:250:kworker.0 00000800:00000100:1.0:1775766209.222204:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775766209.222214:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766209.222232:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766209.222250:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766209.222276:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766209/real 1775766209] req@ffff8f9970a77480 x1862022967199360/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766209.222322:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967199488/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766209.222377:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775766209.222383:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967199360/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766209.222384:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766209.222391:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766209.222410:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2830 00000100:00080000:1.0:1775766209.222415:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766209.222416:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766209.222431:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00080000:0.0:1775766209.222432:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766209.222445:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967199360:192.168.202.146@tcp:250:kworker.0 00000008:00100000:0.0:1775766209.222446:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766209.222477:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967199616/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766209.222491:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775766209.222495:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967199488:192.168.202.146@tcp:38:kworker.0 00000100:00080000:0.0:1775766209.222500:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766209.222507:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766209.222515:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2830 00000100:00080000:0.0:1775766209.222519:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766209.222523:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766209.222527:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766209.222543:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967199744/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766209.222556:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766209.222598:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766209/real 1775766209] req@ffff8f9970a77800 x1862022967199488/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766209.222612:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967199488/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766209.222620:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766209.222624:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766209.222627:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967199488:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766209.222647:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967199616:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766209.222666:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766209.222674:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766209/real 1775766209] req@ffff8f9970a76a00 x1862022967199616/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766209.222683:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967199616/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766209.222711:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766209.222714:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766209.222717:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967199616:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766209.222722:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967199744:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766209.222778:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766209.222789:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766209/real 1775766209] req@ffff8f9970a75500 x1862022967199744/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766209.222798:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967199744/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766209.222805:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766209.222808:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766209.222810:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967199744:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766210.117879:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766210.118063:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766211.141973:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766211.142116:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766212.165876:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766212.165887:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766213.189879:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766213.189942:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766214.213849:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766214.213938:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766214.341836:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766214.341844:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766214.341849:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766214.341858:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2835 00000100:00080000:3.0:1775766214.341869:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766214.341875:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766214.341911:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967199872/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766214.341940:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766214.341943:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766214.341945:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766214.341948:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2835 00000100:00080000:3.0:1775766214.341951:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766214.341953:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766214.341962:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766214.341975:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967200000/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766214.341989:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766214.341993:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766214.341994:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766214.341999:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2835 00000100:00100000:1.0:1775766214.342003:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967199872:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775766214.342022:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766214.342030:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766214.342037:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766214.342048:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967200128/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766214.342055:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766214.342058:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766214.342059:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766214.342062:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2835 00000100:00080000:3.0:1775766214.342066:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766214.342068:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766214.342071:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766214.342077:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967200256/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766214.342092:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766214.342144:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766214/real 1775766214] req@ffff8f9970ede300 x1862022967199872/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766214.342160:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967199872/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766214.342297:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766214.342304:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766214.342309:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967199872:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766214.342343:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967200000:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766214.342396:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766214.342417:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766214/real 1775766214] req@ffff8f9970edf480 x1862022967200000/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766214.342428:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967200000/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766214.342434:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766214.342437:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766214.342439:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967200000:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766214.342443:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967200128:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766214.342454:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766214.342460:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766214/real 1775766214] req@ffff8f9970edd880 x1862022967200128/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766214.342468:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967200128/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766214.342473:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766214.342475:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766214.342477:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967200128:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766214.342514:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967200256:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766214.342537:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766214.342549:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766214/real 1775766214] req@ffff8f9970edf800 x1862022967200256/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766214.342562:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967200256/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766214.342570:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766214.342573:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766214.342575:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967200256:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766215.237834:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766215.237910:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766216.261932:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766216.262071:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775766217.285839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766217.285931:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766218.309874:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766218.309938:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766219.333889:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766219.334011:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766219.461944:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766219.462005:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766219.462012:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766219.462026:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2841 00000100:00080000:0.0:1775766219.462041:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766219.462052:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766219.462123:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967200384/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766219.462169:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766219.462174:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766219.462177:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766219.462182:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2841 00000100:00080000:0.0:1775766219.462188:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766219.462193:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766219.462206:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766219.462234:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967200512/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766219.462249:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766219.462254:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766219.462256:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766219.462262:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2841 00000100:00100000:1.0:1775766219.462270:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967200384:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775766219.462322:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766219.462329:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766219.462336:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766219.462351:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967200640/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766219.462363:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766219.462370:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:1.0:1775766219.462371:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775766219.462374:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766219.462381:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2841 00000100:00080000:0.0:1775766219.462387:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766219.462391:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766219.462397:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766219.462411:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967200768/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766219.462426:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766219/real 1775766219] req@ffff8f9970a77100 x1862022967200384/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766219.462450:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967200384/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766219.462461:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766219.462465:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766219.462470:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967200384:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766219.462683:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967200512:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766219.462897:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766219.462939:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766219/real 1775766219] req@ffff8f9970a76300 x1862022967200512/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766219.463017:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967200512/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766219.463029:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766219.463033:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766219.463037:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967200512:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766219.463092:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967200640:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766219.463121:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766219.463133:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766219/real 1775766219] req@ffff8f9970a75180 x1862022967200640/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766219.463146:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967200640/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766219.463155:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766219.463159:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766219.463173:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967200640:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766219.463180:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967200768:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766219.463215:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766219.463224:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766219/real 1775766219] req@ffff8f9970a75500 x1862022967200768/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766219.463262:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967200768/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766219.463270:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766219.463273:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766219.463276:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967200768:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766220.357864:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766220.358015:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766221.381880:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766221.382015:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766222.405817:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766222.405888:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766223.430010:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766223.430013:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766224.453822:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766224.453891:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766224.581866:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766224.581874:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766224.581878:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766224.581887:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2846 00000100:00080000:2.0:1775766224.581897:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766224.581903:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766224.581954:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2a00 x1862022967200896/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766224.581978:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766224.581981:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766224.581983:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766224.581987:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2846 00000100:00080000:2.0:1775766224.581990:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766224.581994:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766224.582005:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766224.582016:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1880 x1862022967201024/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766224.582021:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967200896:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775766224.582027:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766224.582030:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766224.582055:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766224.582060:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2846 00000100:00080000:2.0:1775766224.582062:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766224.582065:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766224.582070:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766224.582084:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766224.582085:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3b80 x1862022967201152/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766224.582098:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766224.582101:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766224.582103:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766224.582109:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2846 00000100:00080000:2.0:1775766224.582115:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766224.582118:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766224.582122:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766224.582135:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1c00 x1862022967201280/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766224.582142:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766224/real 1775766224] req@ffff8f99703d2a00 x1862022967200896/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766224.582160:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2a00 x1862022967200896/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766224.582171:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766224.582175:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766224.582179:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967200896:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766224.582197:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967201024:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775766224.582221:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766224.582235:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766224/real 1775766224] req@ffff8f99703d1880 x1862022967201024/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766224.582244:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1880 x1862022967201024/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766224.582250:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766224.582253:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766224.582255:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967201024:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775766224.582259:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967201152:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766224.582269:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766224.582274:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766224/real 1775766224] req@ffff8f99703d3b80 x1862022967201152/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766224.582281:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3b80 x1862022967201152/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766224.582286:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766224.582288:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766224.582290:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967201152:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766224.582293:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967201280:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766224.582303:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766224.582309:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766224/real 1775766224] req@ffff8f99703d1c00 x1862022967201280/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766224.582317:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1c00 x1862022967201280/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766224.582322:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766224.582324:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766224.582327:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967201280:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766225.477862:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766225.477978:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766226.501868:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766226.501880:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766227.525864:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766227.525920:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766228.549856:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766228.549937:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766229.573883:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766229.573898:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766229.701939:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766229.701952:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766229.701958:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766229.701972:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2851 00000100:00080000:0.0:1775766229.701987:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766229.701996:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766229.702049:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967201408/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766229.702090:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766229.702094:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766229.702097:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766229.702102:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2851 00000100:00080000:0.0:1775766229.702106:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766229.702110:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766229.702122:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766229.702148:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967201408:192.168.202.146@tcp:250:kworker.0 00000800:00000100:1.0:1775766229.702230:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766229.702239:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967201536/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766229.702253:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766229.702258:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766229.702260:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766229.702264:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2851 00000100:00080000:0.0:1775766229.702269:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766229.702273:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766229.702280:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766229.702287:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766229/real 1775766229] req@ffff8f9970a76a00 x1862022967201408/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766229.702295:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967201664/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766229.702307:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775766229.702310:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967201408/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766229.702319:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766229.702325:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766229.702328:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00080000:0.0:1775766229.702330:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775766229.702333:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967201408:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775766229.702339:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2851 00000100:00080000:0.0:1775766229.702345:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766229.702349:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766229.702355:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766229.702363:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967201536:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775766229.702371:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967201792/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766229.702386:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766229.702398:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766229/real 1775766229] req@ffff8f9970a77800 x1862022967201536/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766229.702409:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967201536/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766229.702415:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766229.702417:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766229.702419:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967201536:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766229.702425:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967201664:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766229.702437:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766229.702443:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766229/real 1775766229] req@ffff8f9970a77480 x1862022967201664/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766229.702450:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967201664/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766229.702456:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766229.702458:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766229.702459:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967201664:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766229.702467:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967201792:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766229.702478:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766229.702484:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766229/real 1775766229] req@ffff8f9970a74380 x1862022967201792/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766229.702496:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967201792/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766229.702503:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766229.702505:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766229.702507:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967201792:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766230.597897:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766230.597921:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766231.621861:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766231.621954:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766232.645951:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766232.646089:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766233.669851:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766233.669903:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766234.693906:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766234.693971:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766234.821913:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766234.821943:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766234.821949:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766234.821964:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2856 00000100:00080000:1.0:1775766234.821989:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766234.821998:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766234.822060:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967201920/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766234.822092:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766234.822095:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766234.822097:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766234.822100:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2856 00000100:00080000:1.0:1775766234.822104:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766234.822107:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766234.822128:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766234.822141:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967202048/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766234.822149:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766234.822152:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766234.822154:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766234.822157:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2856 00000100:00080000:1.0:1775766234.822159:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766234.822164:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766234.822171:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967201920:192.168.202.146@tcp:250:kworker.0 00000008:00100000:1.0:1775766234.822171:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766234.822189:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967202176/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766234.822197:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766234.822200:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766234.822201:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766234.822203:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2856 00000100:00080000:1.0:1775766234.822205:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766234.822207:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766234.822210:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766234.822215:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967202304/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766234.822303:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766234.822357:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766234/real 1775766234] req@ffff8f994f2e5180 x1862022967201920/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766234.822376:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967201920/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766234.822385:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766234.822389:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766234.822392:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967201920:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766234.822398:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967202048:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775766234.822418:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766234.822427:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766234/real 1775766234] req@ffff8f994f2e7800 x1862022967202048/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766234.822435:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967202048/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766234.822441:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766234.822443:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766234.822445:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967202048:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775766234.822466:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967202176:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766234.822478:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766234.822502:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766234/real 1775766234] req@ffff8f994f2e4a80 x1862022967202176/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766234.822514:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967202176/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766234.822519:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766234.822521:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766234.822522:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967202176:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766234.822527:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967202304:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766234.822537:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766234.822543:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766234/real 1775766234] req@ffff8f994f2e6d80 x1862022967202304/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766234.822549:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967202304/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766234.822554:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766234.822556:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766234.822558:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967202304:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766235.717916:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766235.717999:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766236.741901:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766236.741969:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766237.765987:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766237.766017:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766238.789888:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766238.789958:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766239.813861:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766239.813954:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766239.941901:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766239.941909:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766239.941914:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766239.941922:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2861 00000100:00080000:0.0:1775766239.941934:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766239.941940:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766239.941981:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967202432/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766239.942012:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766239.942017:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766239.942019:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766239.942026:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2861 00000100:00080000:0.0:1775766239.942031:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766239.942037:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766239.942044:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967202432:192.168.202.146@tcp:250:kworker.0 00000008:00100000:0.0:1775766239.942049:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766239.942064:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967202560/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766239.942090:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766239.942094:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766239.942095:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766239.942099:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2861 00000100:00080000:0.0:1775766239.942119:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766239.942122:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766239.942128:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775766239.942129:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766239.942149:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967202688/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766239.942163:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766239.942167:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766239.942170:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775766239.942176:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766239/real 1775766239] req@ffff8f9970a75c00 x1862022967202432/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766239.942193:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967202432/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766239.942203:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766239.942207:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766239.942215:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967202432:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775766239.942220:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2861 00000100:00080000:0.0:1775766239.942227:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766239.942231:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766239.942237:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766239.942244:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967202560:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775766239.942254:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967202816/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766239.942274:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766239.942295:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766239/real 1775766239] req@ffff8f9970a74700 x1862022967202560/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766239.942313:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967202560/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766239.942324:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766239.942328:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766239.942332:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967202560:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766239.942341:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967202688:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766239.942367:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766239.942390:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766239/real 1775766239] req@ffff8f9970a76d80 x1862022967202688/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766239.942455:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967202688/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766239.942463:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766239.942466:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766239.942469:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967202688:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766239.942482:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967202816:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766239.942502:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766239.942528:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766239/real 1775766239] req@ffff8f9970a75c00 x1862022967202816/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766239.942537:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967202816/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766239.942543:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766239.942545:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766239.942547:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967202816:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766240.837881:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766240.837956:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766241.861942:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766241.861959:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766242.885905:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766242.885998:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766243.909884:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766243.909902:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766244.933901:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766244.933964:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00080000:3.0:1775766245.061874:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766245.061883:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766245.061888:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766245.061900:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2866 00000100:00080000:3.0:1775766245.061914:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766245.061922:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766245.061961:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967202944/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766245.061997:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766245.062001:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766245.062004:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766245.062009:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2866 00000100:00080000:3.0:1775766245.062014:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766245.062019:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766245.062024:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967202944:192.168.202.146@tcp:250:kworker.0 00000008:00100000:3.0:1775766245.062031:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766245.062049:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967203072/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766245.062062:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766245.062066:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766245.062068:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766245.062072:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2866 00000100:00080000:3.0:1775766245.062077:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766245.062082:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766245.062088:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766245.062102:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967203200/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766245.062133:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775766245.062134:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967203072:192.168.202.146@tcp:38:kworker.0 00000100:00080000:3.0:1775766245.062140:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766245.062143:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766245.062150:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2866 00000100:00080000:3.0:1775766245.062154:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766245.062160:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766245.062176:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766245.062177:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967203200:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766245.062194:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967203328/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766245.062249:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967203328:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766245.957909:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766245.957911:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766246.981828:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766246.981970:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766248.005839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766248.005873:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775766248.134080:0:2367:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:1.0:1775766248.134148:0:2367:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775766248.134166:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775766248.134207:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775766248.134218:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775766248.134227:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:0.0:1775766248.134392:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766245/real 1775766248] req@ffff8f9970edfb80 x1862022967203328/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766248.134424:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967203328/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766248.134440:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766248.134446:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766248.134451:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967203328:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766248.134461:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766245/real 1775766248] req@ffff8f9970edc380 x1862022967203200/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766248.134476:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967203200/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766248.134485:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766248.134490:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766248.134525:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967203200:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766248.134533:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766245/real 1775766248] req@ffff8f9970edca80 x1862022967203072/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766248.134548:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967203072/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766248.134556:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766248.134560:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766248.134563:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967203072:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775766248.134571:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766245/real 1775766248] req@ffff8f9970edea00 x1862022967202944/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766248.134584:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967202944/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766248.134591:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766248.134595:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766248.134598:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967202944:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766249.029826:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766249.029923:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766250.053830:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766250.053888:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766250.181868:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766250.181879:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766250.181884:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766250.181911:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2871 00000100:00080000:3.0:1775766250.181924:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766250.181931:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766250.181968:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddf80 x1862022967203456/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766250.182009:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766250.182025:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766250.182027:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766250.182032:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2871 00000100:00080000:3.0:1775766250.182036:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766250.182041:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766250.182052:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766250.182068:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967203456:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766250.182070:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd180 x1862022967203584/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766250.182088:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766250.182093:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766250.182095:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766250.182100:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2871 00000100:00080000:3.0:1775766250.182106:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766250.182111:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766250.182118:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766250.182133:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967203712/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766250.182146:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1775766250.182150:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775766250.182151:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766250.182157:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766250.182163:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2871 00000100:00080000:3.0:1775766250.182169:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766250.182174:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766250.182181:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766250.182195:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967203840/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766250.182205:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766250/real 1775766250] req@ffff8f9970eddf80 x1862022967203456/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766250.182248:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddf80 x1862022967203456/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766250.182269:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766250.182275:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766250.182280:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967203456:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766250.182309:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967203584:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775766250.182338:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766250.182353:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766250/real 1775766250] req@ffff8f9970edd180 x1862022967203584/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766250.182369:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd180 x1862022967203584/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766250.182379:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766250.182383:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766250.182386:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967203584:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775766250.182393:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967203712:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766250.182418:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766250.182431:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766250/real 1775766250] req@ffff8f9970edea00 x1862022967203712/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766250.182444:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967203712/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766250.182454:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766250.182457:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766250.182460:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967203712:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766250.182468:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967203840:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766250.182492:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766250.182503:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766250/real 1775766250] req@ffff8f9970edca80 x1862022967203840/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766250.182516:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967203840/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766250.182525:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766250.182529:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766250.182533:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967203840:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766251.077810:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766251.077855:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766252.101931:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766252.101944:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766253.125826:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766253.125892:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766254.149870:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766254.149945:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766255.173899:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766255.174002:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766255.301920:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766255.301944:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766255.301948:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766255.301958:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2876 00000100:00080000:0.0:1775766255.301969:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766255.301976:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766255.302028:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967203968/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766255.302053:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766255.302057:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766255.302074:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766255.302079:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2876 00000100:00080000:0.0:1775766255.302112:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766255.302117:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766255.302125:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967203968:192.168.202.146@tcp:250:kworker.0 00000008:00100000:0.0:1775766255.302127:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766255.302153:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967204096/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766255.302165:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766255.302168:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766255.302171:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766255.302201:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2876 00000100:00080000:0.0:1775766255.302206:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766255.302210:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766255.302215:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775766255.302217:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766255.302237:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967204224/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766255.302247:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766255.302251:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766255.302253:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766255.302256:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2876 00000100:00080000:0.0:1775766255.302260:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766255.302264:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766255.302270:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766255.302270:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766255/real 1775766255] req@ffff8f9970a74380 x1862022967203968/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766255.302280:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967204352/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766255.302313:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967203968/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766255.302325:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766255.302330:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766255.302334:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967203968:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766255.302363:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967204096:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766255.302383:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766255.302408:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766255/real 1775766255] req@ffff8f9970a77480 x1862022967204096/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766255.302417:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967204096/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766255.302424:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766255.302426:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766255.302428:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967204096:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766255.302432:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967204224:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766255.302447:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766255.302453:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766255/real 1775766255] req@ffff8f9970a77800 x1862022967204224/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766255.302461:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967204224/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766255.302466:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766255.302468:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766255.302470:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967204224:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766255.302474:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967204352:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766255.302484:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766255.302489:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766255/real 1775766255] req@ffff8f9970a76a00 x1862022967204352/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766255.302497:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967204352/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766255.302513:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766255.302516:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766255.302518:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967204352:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766256.197838:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766256.197943:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766257.221839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766257.221885:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766258.245819:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766258.245880:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766259.269869:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766259.269873:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766260.293854:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766260.294024:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766260.421830:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766260.421839:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766260.421843:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766260.421852:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2882 00000100:00080000:0.0:1775766260.421863:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766260.421869:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766260.421921:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967204480/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766260.421951:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766260.421954:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766260.421956:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766260.421960:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2882 00000100:00080000:0.0:1775766260.421963:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766260.421966:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766260.421975:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766260.421986:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967204608/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766260.421995:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766260.421998:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766260.422000:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766260.422002:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2882 00000100:00080000:0.0:1775766260.422007:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775766260.422012:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967204480:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775766260.422013:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766260.422023:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766260.422031:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967204736/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766260.422040:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766260.422043:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766260.422045:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766260.422048:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2882 00000100:00080000:0.0:1775766260.422051:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766260.422054:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766260.422057:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766260.422066:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967204864/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766260.422085:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766260.422130:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766260/real 1775766260] req@ffff8f9970a75500 x1862022967204480/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766260.422149:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967204480/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766260.422159:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766260.422164:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766260.422168:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967204480:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766260.422183:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967204608:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766260.422206:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766260.422222:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766260/real 1775766260] req@ffff8f9970a75180 x1862022967204608/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766260.422232:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967204608/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766260.422287:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766260.422291:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766260.422294:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967204608:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766260.422320:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967204736:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766260.422349:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766260.422378:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766260/real 1775766260] req@ffff8f9970a76300 x1862022967204736/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766260.422393:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967204736/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766260.422404:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766260.422409:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766260.422412:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967204736:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766260.422420:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967204864:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766260.422442:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766260.422455:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766260/real 1775766260] req@ffff8f9970a77100 x1862022967204864/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766260.422469:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967204864/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766260.422479:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766260.422483:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766260.422506:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967204864:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766261.317878:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766261.317962:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766262.341852:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766262.341945:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766263.365861:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766263.365942:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766264.389834:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766264.389914:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766265.413895:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766265.414041:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766265.541948:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766265.541962:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766265.541971:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766265.541987:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2887 00000100:00080000:3.0:1775766265.542028:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766265.542039:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766265.542093:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967204992/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766265.542134:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766265.542138:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766265.542141:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766265.542146:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2887 00000100:00080000:3.0:1775766265.542151:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766265.542156:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766265.542167:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766265.542183:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967205120/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766265.542194:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766265.542198:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766265.542200:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766265.542205:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2887 00000100:00080000:3.0:1775766265.542209:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766265.542212:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766265.542219:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766265.542222:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967204992:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766265.542233:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967205248/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766265.542254:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766265.542257:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766265.542259:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766265.542262:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2887 00000100:00080000:3.0:1775766265.542267:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766265.542270:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766265.542276:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766265.542288:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967205376/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766265.542321:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766265.542384:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766265/real 1775766265] req@ffff8f9970edc380 x1862022967204992/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766265.542412:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967204992/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766265.542424:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766265.542429:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766265.542434:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967204992:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766265.542442:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967205120:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766265.542471:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766265.542485:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766265/real 1775766265] req@ffff8f9970edfb80 x1862022967205120/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766265.542498:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967205120/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766265.542506:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766265.542509:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766265.542513:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967205120:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766265.542617:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967205248:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766265.542643:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766265.542656:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766265/real 1775766265] req@ffff8f9970edf800 x1862022967205248/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766265.542669:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967205248/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766265.542678:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766265.542682:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766265.542686:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967205248:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766265.542696:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967205376:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766265.542724:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766265.542799:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766265/real 1775766265] req@ffff8f9970edd880 x1862022967205376/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766265.542818:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967205376/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766265.542848:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766265.542852:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766265.542856:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967205376:192.168.202.146@tcp:8:kworker.0 00010000:00010000:0.0:1775766266.437865:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766266.437869:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775766267.461878:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766267.461976:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766268.485851:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766268.485959:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766269.509863:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766269.510009:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766270.533845:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766270.533944:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766270.662228:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766270.662246:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766270.662253:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766270.662270:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2892 00000100:00080000:0.0:1775766270.662290:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766270.662303:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766270.662372:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967205504/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766270.662816:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775766270.662819:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967205504:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775766270.662823:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766270.662828:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766270.662838:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2892 00000100:00080000:0.0:1775766270.662850:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766270.662857:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766270.662871:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766270.662921:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967205632/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766270.662937:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766270.662940:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766270.662942:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766270.662947:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2892 00000100:00080000:0.0:1775766270.662952:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766270.662956:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766270.662962:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766270.662974:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967205760/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766270.662986:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766270.662990:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:1.0:1775766270.662996:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775766270.662997:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766270.663009:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2892 00000100:00080000:0.0:1775766270.663017:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766270.663024:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766270.663031:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766270.663047:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967205888/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766270.663062:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766270/real 1775766270] req@ffff8f9970a77100 x1862022967205504/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766270.663087:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967205504/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766270.663099:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766270.663106:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766270.663113:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967205504:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766270.663154:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967205632:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766270.663192:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766270.663213:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766270/real 1775766270] req@ffff8f9970a76300 x1862022967205632/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766270.663234:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967205632/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766270.663247:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766270.663252:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766270.663258:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967205632:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766270.663269:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967205760:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766270.663299:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766270.663313:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766270/real 1775766270] req@ffff8f9970a75180 x1862022967205760/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766270.663370:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967205760/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766270.663383:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766270.663388:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766270.663393:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967205760:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766270.663403:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967205888:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766270.663434:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766270.663459:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766270/real 1775766270] req@ffff8f9970a75500 x1862022967205888/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766270.663491:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967205888/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766270.663503:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766270.663507:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766270.663510:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967205888:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766271.557857:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766271.557924:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766272.581814:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766272.581870:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766273.605932:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766273.605994:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766274.629836:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766274.629893:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766275.653875:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766275.653986:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766275.781847:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766275.781859:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766275.781864:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766275.781875:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2897 00000100:00080000:0.0:1775766275.781887:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766275.781894:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766275.781934:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967206016/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766275.781965:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766275.781970:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766275.781973:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766275.781978:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2897 00000100:00080000:0.0:1775766275.781983:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766275.781987:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766275.781998:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766275.781999:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967206016:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766275.782019:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967206144/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766275.782039:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766275.782043:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766275.782045:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766275.782049:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2897 00000100:00080000:0.0:1775766275.782054:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1775766275.782059:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775766275.782062:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766275.782069:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766275.782083:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967206272/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766275.782095:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766275.782100:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1775766275.782100:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766275/real 1775766275] req@ffff8f9970a76a00 x1862022967206016/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766275.782102:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766275.782110:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2897 00000100:00080000:0.0:1775766275.782118:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766275.782123:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766275.782127:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766275.782138:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967206400/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766275.782178:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967206016/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766275.782185:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766275.782188:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766275.782191:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967206016:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766275.782229:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967206144:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766275.782254:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766275.782269:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766275/real 1775766275] req@ffff8f9970a77800 x1862022967206144/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766275.782281:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967206144/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766275.782287:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766275.782290:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766275.782294:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967206144:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766275.782301:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967206272:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766275.782322:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766275.782349:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766275/real 1775766275] req@ffff8f9970a77480 x1862022967206272/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766275.782362:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967206272/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766275.782392:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766275.782394:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766275.782397:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967206272:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766275.782401:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967206400:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766275.782416:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766275.782449:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766275/real 1775766275] req@ffff8f9970a74380 x1862022967206400/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766275.782457:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967206400/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766275.782463:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766275.782465:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766275.782467:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967206400:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766276.677882:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766276.677909:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766277.701870:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766277.701952:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766278.725859:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766278.725942:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766279.749866:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766279.749960:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766280.773910:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766280.773939:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766280.901841:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766280.901855:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766280.901860:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766280.901875:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2902 00000100:00080000:2.0:1775766280.901889:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766280.901896:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766280.901940:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2300 x1862022967206528/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766280.901969:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766280.901972:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766280.901973:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766280.901977:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2902 00000100:00080000:2.0:1775766280.901979:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766280.901982:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766280.901992:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766280.902064:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967206528:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766280.902138:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3480 x1862022967206656/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766280.902139:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775766280.902155:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766280.902159:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766280.902161:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766280.902166:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2902 00000100:00080000:2.0:1775766280.902172:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766280.902177:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766280.902184:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766280.902192:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766280/real 1775766280] req@ffff8f99703d2300 x1862022967206528/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766280.902196:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3100 x1862022967206784/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766280.902217:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766280.902219:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766280.902221:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775766280.902225:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2300 x1862022967206528/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766280.902228:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2902 00000100:00080000:2.0:1775766280.902233:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766280.902239:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766280.902243:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766280.902249:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1180 x1862022967206912/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766280.902278:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766280.902282:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766280.902285:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967206528:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766280.902303:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967206656:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766280.902320:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766280.902329:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766280/real 1775766280] req@ffff8f99703d3480 x1862022967206656/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766280.902337:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3480 x1862022967206656/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766280.902342:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766280.902345:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766280.902347:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967206656:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766280.902350:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967206784:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766280.902361:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766280.902366:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766280/real 1775766280] req@ffff8f99703d3100 x1862022967206784/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766280.902402:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3100 x1862022967206784/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766280.902407:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766280.902409:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766280.902412:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967206784:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766280.902425:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967206912:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766280.902440:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766280.902447:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766280/real 1775766280] req@ffff8f99703d1180 x1862022967206912/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766280.902455:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1180 x1862022967206912/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766280.902460:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766280.902462:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766280.902464:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967206912:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766281.797876:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766281.797911:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766282.821848:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766282.821999:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766283.845823:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766283.845879:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766284.869818:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766284.869881:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766285.893861:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766285.893906:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766286.021963:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766286.021974:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766286.021980:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766286.021990:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2907 00000100:00080000:1.0:1775766286.022003:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766286.022010:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766286.022075:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967207040/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766286.022109:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766286.022112:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766286.022114:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766286.022119:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2907 00000100:00080000:1.0:1775766286.022122:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766286.022125:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766286.022152:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766286.022169:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967207168/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766286.022179:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766286.022182:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766286.022184:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766286.022189:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2907 00000100:00080000:1.0:1775766286.022193:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766286.022197:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766286.022204:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766286.022216:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967207040:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766286.022216:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967207296/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766286.022241:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766286.022243:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766286.022245:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766286.022259:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2907 00000100:00080000:1.0:1775766286.022261:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766286.022264:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766286.022267:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766286.022273:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967207424/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766286.022415:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766286.022460:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766286/real 1775766286] req@ffff8f994f2e6d80 x1862022967207040/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766286.022483:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967207040/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766286.022508:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766286.022513:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766286.022516:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967207040:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766286.022524:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967207168:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775766286.022546:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766286.022557:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766286/real 1775766286] req@ffff8f994f2e4a80 x1862022967207168/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766286.022567:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967207168/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766286.022581:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766286.022584:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766286.022587:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967207168:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775766286.022610:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967207296:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766286.022626:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766286.022645:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766286/real 1775766286] req@ffff8f994f2e7800 x1862022967207296/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766286.022667:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967207296/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766286.022677:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766286.022681:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766286.022685:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967207296:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766286.022693:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967207424:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766286.022715:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766286.022724:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766286/real 1775766286] req@ffff8f994f2e5180 x1862022967207424/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766286.022733:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967207424/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766286.022783:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766286.022788:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766286.022793:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967207424:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766286.917841:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766286.917881:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766287.941875:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766287.941893:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766288.965868:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766288.965977:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766289.989885:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766289.989933:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775766291.013853:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766291.013929:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766291.141839:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766291.141847:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766291.141851:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766291.141860:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2912 00000100:00080000:3.0:1775766291.141870:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766291.141885:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766291.141922:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967207552/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766291.141944:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766291.141947:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766291.141948:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766291.141952:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2912 00000100:00080000:3.0:1775766291.141954:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766291.141956:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766291.141965:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766291.141973:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967207680/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766291.141980:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766291.141983:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766291.141984:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766291.141987:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2912 00000100:00080000:3.0:1775766291.141989:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766291.141994:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766291.141996:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967207552:192.168.202.146@tcp:250:kworker.0 00000008:00100000:3.0:1775766291.141998:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766291.142009:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967207808/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766291.142016:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766291.142018:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766291.142019:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766291.142022:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2912 00000100:00080000:3.0:1775766291.142024:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766291.142027:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766291.142030:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766291.142037:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967207936/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766291.142063:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766291.142102:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766291/real 1775766291] req@ffff8f9970edf480 x1862022967207552/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766291.142119:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967207552/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766291.142174:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766291.142177:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766291.142180:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967207552:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766291.142186:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967207680:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775766291.142202:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766291.142211:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766291/real 1775766291] req@ffff8f9970ede300 x1862022967207680/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766291.142219:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967207680/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766291.142232:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766291.142234:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766291.142236:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967207680:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775766291.142255:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967207808:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766291.142286:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766291.142298:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766291/real 1775766291] req@ffff8f9970edc000 x1862022967207808/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766291.142311:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967207808/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766291.142332:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766291.142336:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766291.142340:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967207808:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766291.142347:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967207936:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766291.142362:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766291.142370:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766291/real 1775766291] req@ffff8f9970ede680 x1862022967207936/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766291.142377:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967207936/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766291.142383:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766291.142385:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766291.142387:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967207936:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766292.037855:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766292.037904:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766293.061837:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766293.061925:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766294.085820:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766294.085894:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766295.109891:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766295.109916:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766296.133837:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766296.133959:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766296.261938:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766296.261947:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766296.261952:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766296.261963:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2917 00000100:00080000:2.0:1775766296.261976:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766296.261984:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766296.262027:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1500 x1862022967208064/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766296.262056:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766296.262060:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766296.262065:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766296.262075:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2917 00000100:00080000:2.0:1775766296.262081:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766296.262088:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766296.262100:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766296.262119:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967208064:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766296.262169:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0700 x1862022967208192/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766296.262183:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766296.262187:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766296.262189:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766296.262193:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2917 00000100:00080000:2.0:1775766296.262197:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766296.262203:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766296.262210:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766296.262219:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0e00 x1862022967208320/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766296.262264:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775766296.262303:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766296.262307:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766296.262309:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766296.262316:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2917 00000100:00080000:2.0:1775766296.262323:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766296.262329:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766296.262336:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766296/real 1775766296] req@ffff8f99703d1500 x1862022967208064/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1775766296.262341:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766296.262377:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2d80 x1862022967208448/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766296.262386:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1500 x1862022967208064/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766296.262430:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766296.262443:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766296.262456:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967208064:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766296.262505:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967208192:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775766296.262555:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766296.262595:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766296/real 1775766296] req@ffff8f99703d0700 x1862022967208192/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766296.262630:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0700 x1862022967208192/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766296.262650:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766296.262663:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766296.262674:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967208192:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775766296.262694:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967208320:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766296.262852:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766296.262871:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766296/real 1775766296] req@ffff8f99703d0e00 x1862022967208320/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766296.262886:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0e00 x1862022967208320/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766296.262895:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766296.262899:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766296.262902:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967208320:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766296.262909:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967208448:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766296.262932:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766296.262942:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766296/real 1775766296] req@ffff8f99703d2d80 x1862022967208448/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766296.262954:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2d80 x1862022967208448/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766296.262964:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766296.262967:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766296.262970:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967208448:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766297.157855:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766297.157949:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766298.181839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766298.181927:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766299.205884:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766299.206005:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766300.229844:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766300.229910:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766301.253869:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766301.253984:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766301.381926:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766301.381935:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766301.381940:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766301.381948:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2922 00000100:00080000:1.0:1775766301.381957:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766301.381963:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766301.381999:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967208576/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766301.382025:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766301.382028:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766301.382030:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766301.382033:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2922 00000100:00080000:1.0:1775766301.382035:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766301.382039:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766301.382048:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766301.382060:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967208704/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766301.382077:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967208576:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775766301.382080:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766301.382097:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766301.382099:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766301.382103:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2922 00000100:00080000:1.0:1775766301.382105:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766301.382108:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766301.382112:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766301.382127:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967208832/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766301.382134:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766301.382137:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766301.382139:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766301.382142:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2922 00000100:00080000:1.0:1775766301.382145:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766301.382147:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766301.382152:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766301.382159:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967208960/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766301.382213:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766301.382289:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766301/real 1775766301] req@ffff8f994f2e4000 x1862022967208576/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766301.382309:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967208576/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766301.382319:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766301.382323:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766301.382326:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967208576:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766301.382352:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967208704:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775766301.382371:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766301.382381:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766301/real 1775766301] req@ffff8f994f2e6a00 x1862022967208704/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766301.382389:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967208704/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766301.382400:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766301.382402:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766301.382404:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967208704:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775766301.382408:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967208832:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766301.382420:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766301.382442:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766301/real 1775766301] req@ffff8f994f2e5f80 x1862022967208832/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766301.382450:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967208832/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766301.382456:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766301.382458:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766301.382460:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967208832:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766301.382465:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967208960:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766301.382477:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766301.382483:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766301/real 1775766301] req@ffff8f994f2e6680 x1862022967208960/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766301.382491:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967208960/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766301.382496:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766301.382498:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766301.382595:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967208960:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766302.277992:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766302.278014:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766303.301901:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766303.301977:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766304.325919:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766304.325925:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775766305.349812:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766305.349926:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766306.373841:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766306.373926:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766306.501845:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766306.501856:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766306.501862:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766306.501876:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2928 00000100:00080000:1.0:1775766306.501890:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766306.501898:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766306.501940:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967209088/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766306.501975:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766306.501979:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766306.501981:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766306.501986:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2928 00000100:00080000:1.0:1775766306.501991:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766306.501997:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766306.502008:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766306.502019:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967209088:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766306.502027:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967209216/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766306.502043:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766306.502048:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766306.502050:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766306.502055:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2928 00000100:00080000:1.0:1775766306.502061:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766306.502065:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766306.502071:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766306.502087:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967209344/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766306.502110:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775766306.502179:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766306.502183:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766306.502185:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766306.502189:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2928 00000100:00080000:1.0:1775766306.502193:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766306.502197:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766306.502202:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766306.502230:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967209472/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766306.502293:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766306/real 1775766306] req@ffff8f994f2e4700 x1862022967209088/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766306.502344:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967209088/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766306.502357:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766306.502362:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766306.502366:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967209088:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766306.502389:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967209216:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775766306.502411:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766306.502421:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766306/real 1775766306] req@ffff8f994f2e6300 x1862022967209216/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766306.502435:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967209216/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766306.502443:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766306.502447:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766306.502451:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967209216:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775766306.502461:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967209344:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766306.502479:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766306.502489:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766306/real 1775766306] req@ffff8f994f2e4e00 x1862022967209344/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766306.502502:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967209344/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766306.502511:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766306.502514:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766306.502517:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967209344:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766306.502523:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967209472:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766306.502536:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766306.502544:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766306/real 1775766306] req@ffff8f994f2e7b80 x1862022967209472/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766306.502555:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967209472/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766306.502561:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766306.502564:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766306.502566:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967209472:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766307.397835:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766307.397895:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766308.421862:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766308.421929:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766309.445918:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766309.445957:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766310.469872:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766310.469972:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766311.493819:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766311.493911:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766311.621921:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766311.621937:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766311.621944:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766311.621959:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2933 00000100:00080000:0.0:1775766311.621975:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766311.621984:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766311.622039:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967209600/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766311.622127:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766311.622132:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766311.622134:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766311.622141:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2933 00000100:00080000:0.0:1775766311.622147:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775766311.622153:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967209600:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775766311.622154:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766311.622184:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766311.622206:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967209728/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766311.622221:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766311.622225:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766311.622228:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766311.622232:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2933 00000100:00080000:0.0:1775766311.622241:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766311.622246:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766311.622252:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766311.622265:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967209856/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766311.622279:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766311.622285:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766311.622289:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766311.622294:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2933 00000100:00080000:0.0:1775766311.622300:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766311.622322:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766311.622329:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967209728:192.168.202.146@tcp:38:kworker.0 00000008:00100000:0.0:1775766311.622330:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766311.622358:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967209984/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766311.622389:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967209856:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766311.622427:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967209984:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766312.517865:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766312.517991:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766313.541870:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766313.541939:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766314.565926:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766314.565963:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775766314.694219:0:2369:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:2.0:1775766314.694289:0:2369:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775766314.694322:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:2.0:1775766314.694401:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:2.0:1775766314.694422:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:2.0:1775766314.694439:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:1.0:1775766314.694508:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766311/real 1775766314] req@ffff8f9970a74380 x1862022967209984/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766314.694564:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967209984/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766314.694588:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766314.694793:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766314.694805:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967209984:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766314.694824:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766311/real 1775766314] req@ffff8f9970a76d80 x1862022967209728/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766314.694868:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967209728/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766314.694881:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766314.694889:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766314.694894:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967209728:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766314.694912:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766311/real 1775766314] req@ffff8f9970a74700 x1862022967209856/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766314.694928:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967209856/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766314.694937:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766314.694941:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766314.694944:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967209856:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766314.694951:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766311/real 1775766314] req@ffff8f9970a75c00 x1862022967209600/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766314.694963:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967209600/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766314.694970:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766314.695003:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766314.695007:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967209600:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766315.589891:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766315.590019:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766316.613916:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766316.614100:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766316.741970:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766316.741984:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766316.741992:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766316.742008:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2938 00000100:00080000:0.0:1775766316.742080:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766316.742091:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766316.742149:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967210112/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766316.742208:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766316.742214:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766316.742217:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775766316.742314:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967210112:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775766316.742318:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2938 00000100:00080000:0.0:1775766316.742349:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766316.742358:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766316.742376:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766316.742404:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967210240/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766316.742424:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766316.742429:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766316.742431:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766316.742437:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2938 00000100:00080000:0.0:1775766316.742454:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766316.742461:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766316.742469:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775766316.742475:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766316.742494:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967210368/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766316.742535:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766316.742541:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766316.742545:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766316.742552:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2938 00000100:00080000:0.0:1775766316.742560:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766316.742567:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766316.742583:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766316.742587:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766316/real 1775766316] req@ffff8f9970a77480 x1862022967210112/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766316.742609:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967210496/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766316.742643:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967210112/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766316.742666:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766316.742675:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766316.742682:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967210112:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766316.742725:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967210240:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766316.742811:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766316.742837:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766316/real 1775766316] req@ffff8f9970a77800 x1862022967210240/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766316.742857:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967210240/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766316.742871:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766316.742876:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766316.742882:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967210240:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766316.742894:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967210368:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766316.743361:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766316.743427:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766316/real 1775766316] req@ffff8f9970a76a00 x1862022967210368/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766316.743457:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967210368/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766316.743476:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766316.743485:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766316.743491:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967210368:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766316.743509:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967210496:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766316.743541:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766316.743560:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766316/real 1775766316] req@ffff8f9970a75500 x1862022967210496/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766316.743577:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967210496/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766316.743589:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766316.743594:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766316.743599:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967210496:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766317.637894:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766317.638096:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766318.661920:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766318.662128:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766319.685847:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766319.685980:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766320.709891:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766320.710074:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766321.733911:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766321.734045:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766321.861942:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766321.861959:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766321.861966:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766321.862104:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2943 00000100:00080000:1.0:1775766321.862122:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766321.862136:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766321.862192:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967210624/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766321.862237:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766321.862242:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766321.862245:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766321.862251:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2943 00000100:00080000:1.0:1775766321.862256:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766321.862261:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766321.862274:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766321.862293:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967210752/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766321.862332:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766321.862336:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766321.862338:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766321.862346:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2943 00000100:00080000:1.0:1775766321.862352:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775766321.862361:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967210624:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775766321.862363:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766321.862372:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766321.862390:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967210880/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766321.862445:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766321.862452:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766321.862457:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766321.862467:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2943 00000100:00080000:1.0:1775766321.862489:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766321.862497:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:0.0:1775766321.862509:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:1.0:1775766321.862510:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766321.862540:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967211008/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766321.862591:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766321/real 1775766321] req@ffff8f994f2e7b80 x1862022967210624/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766321.862628:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967210624/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766321.862650:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766321.862659:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766321.862668:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967210624:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766321.862721:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967210752:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775766321.862802:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766321.862823:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766321/real 1775766321] req@ffff8f994f2e4e00 x1862022967210752/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766321.862841:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967210752/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766321.862853:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766321.862858:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766321.862863:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967210752:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775766321.862877:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967210880:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766321.862901:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766321.862912:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766321/real 1775766321] req@ffff8f994f2e6300 x1862022967210880/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766321.862927:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967210880/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766321.862939:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766321.862945:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766321.862951:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967210880:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766321.862963:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967211008:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766321.863042:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766321.863066:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766321/real 1775766321] req@ffff8f994f2e4700 x1862022967211008/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766321.863091:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967211008/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766321.863105:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766321.863110:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766321.863114:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967211008:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766322.757860:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766322.758010:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766323.782067:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766323.782078:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775766324.805802:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766324.805886:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766325.829842:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766325.830077:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766326.853821:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766326.853882:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766326.981892:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766326.981974:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766326.981980:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766326.981995:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2948 00000100:00080000:0.0:1775766326.982010:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766326.982019:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766326.982071:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967211136/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766326.982109:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766326.982113:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766326.982115:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766326.982122:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2948 00000100:00080000:0.0:1775766326.982127:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766326.982132:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766326.982146:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766326.982150:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967211136:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766326.982166:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967211264/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766326.982181:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766326.982186:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766326.982188:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766326.982194:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2948 00000100:00080000:0.0:1775766326.982200:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766326.982207:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766326.982214:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775766326.982231:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766326.982233:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967211392/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766326.982250:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766326.982255:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766326.982257:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766326.982265:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2948 00000100:00080000:0.0:1775766326.982271:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775766326.982277:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766326/real 1775766326] req@ffff8f9970a75180 x1862022967211136/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766326.982279:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766326.982297:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766326.982304:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967211136/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766326.982315:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1775766326.982318:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967211520/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766326.982323:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766326.982329:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967211136:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766326.982363:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967211264:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766326.982393:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766326.982444:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766326/real 1775766326] req@ffff8f9970a76300 x1862022967211264/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766326.982453:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967211264/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766326.982459:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766326.982462:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766326.982464:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967211264:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766326.982469:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967211392:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766326.982485:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766326.982492:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766326/real 1775766326] req@ffff8f9970a77100 x1862022967211392/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766326.982504:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967211392/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766326.982512:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766326.982515:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766326.982518:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967211392:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766326.982525:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967211520:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766326.982546:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766326.982557:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766326/real 1775766326] req@ffff8f9970a75500 x1862022967211520/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766326.982565:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967211520/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766326.982584:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766326.982587:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766326.982589:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967211520:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766327.877853:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766327.877980:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766328.901817:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766328.901890:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766329.925864:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766329.926062:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766330.949909:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766330.949974:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775766331.973820:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766331.973911:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766332.101946:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766332.101953:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766332.101958:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766332.101967:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2953 00000100:00080000:3.0:1775766332.101977:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766332.101984:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766332.102017:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967211648/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766332.102037:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766332.102041:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766332.102042:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766332.102048:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2953 00000100:00080000:3.0:1775766332.102051:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766332.102055:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766332.102063:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766332.102076:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967211776/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766332.102082:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967211648:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775766332.102086:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766332.102092:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766332.102093:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766332.102097:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2953 00000100:00080000:3.0:1775766332.102102:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766332.102105:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766332.102110:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766332.102125:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967211904/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766332.102136:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766332.102139:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766332.102140:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766332.102143:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2953 00000100:00080000:3.0:1775766332.102145:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766332.102148:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:1.0:1775766332.102154:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:3.0:1775766332.102156:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766332.102166:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967212032/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766332.102220:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766332/real 1775766332] req@ffff8f9970edc700 x1862022967211648/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766332.102241:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967211648/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766332.102249:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766332.102252:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766332.102255:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967211648:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766332.102277:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967211776:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766332.102296:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766332.102335:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766332/real 1775766332] req@ffff8f9970edd500 x1862022967211776/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766332.102348:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967211776/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766332.102357:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766332.102414:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766332.102417:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967211776:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766332.102422:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967211904:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766332.102452:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766332.102463:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766332/real 1775766332] req@ffff8f9970eded80 x1862022967211904/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766332.102472:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967211904/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766332.102478:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766332.102482:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766332.102484:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967211904:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766332.102490:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967212032:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766332.102503:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766332.102514:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766332/real 1775766332] req@ffff8f9970edce00 x1862022967212032/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766332.102522:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967212032/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766332.102527:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766332.102529:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766332.102532:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967212032:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766332.997860:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766332.998005:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766334.021906:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766334.021966:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766335.045875:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766335.045945:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766336.069860:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766336.069882:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766337.093860:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766337.093925:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766337.222404:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766337.222417:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766337.222422:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766337.222435:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2958 00000100:00080000:0.0:1775766337.222449:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766337.222458:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766337.222506:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967212160/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766337.222542:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766337.222548:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766337.222550:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766337.222556:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2958 00000100:00080000:0.0:1775766337.222561:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766337.222566:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766337.222577:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766337.222593:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967212288/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766337.222607:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766337.222611:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766337.222613:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766337.222618:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2958 00000100:00080000:0.0:1775766337.222622:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766337.222626:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766337.222632:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766337.222644:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967212416/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766337.222656:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766337.222660:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766337.222662:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766337.222667:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2958 00000100:00080000:0.0:1775766337.222671:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766337.222675:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766337.222772:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967212160:192.168.202.146@tcp:250:kworker.0 00000008:00100000:0.0:1775766337.222861:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775766337.222875:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766337.222887:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967212544/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766337.222916:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766337/real 1775766337] req@ffff8f9970a76a00 x1862022967212160/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766337.222934:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967212160/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766337.222943:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766337.222947:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766337.222950:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967212160:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766337.222957:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967212288:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766337.222986:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766337.223013:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766337/real 1775766337] req@ffff8f9970a77800 x1862022967212288/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766337.223024:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967212288/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766337.223031:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766337.223033:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766337.223036:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967212288:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766337.223040:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967212416:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766337.223085:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766337.223093:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766337/real 1775766337] req@ffff8f9970a77480 x1862022967212416/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766337.223120:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967212416/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766337.223127:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766337.223129:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766337.223132:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967212416:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766337.223157:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967212544:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766337.223181:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766337.223193:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766337/real 1775766337] req@ffff8f9970a75c00 x1862022967212544/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766337.223207:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967212544/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766337.223216:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766337.223218:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766337.223221:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967212544:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766338.117913:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766338.117928:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766339.141892:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766339.141973:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766340.165867:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766340.165933:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766341.189875:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766341.189928:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766342.213871:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766342.213931:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766342.341868:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766342.341877:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766342.341882:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766342.341890:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2963 00000100:00080000:2.0:1775766342.341900:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766342.341905:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766342.341946:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0380 x1862022967212672/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766342.341972:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766342.341974:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766342.341976:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766342.341979:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2963 00000100:00080000:2.0:1775766342.341981:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775766342.342013:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967212672:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775766342.342023:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766342.342031:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766342.342042:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0000 x1862022967212800/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766342.342050:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766342.342053:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766342.342054:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766342.342057:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2963 00000100:00080000:2.0:1775766342.342059:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766342.342062:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766342.342065:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766342.342073:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2680 x1862022967212928/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766342.342077:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775766342.342084:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766342.342087:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766342.342088:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766342.342091:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2963 00000100:00080000:2.0:1775766342.342094:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766342.342096:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766342.342099:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766342.342106:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3800 x1862022967213056/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766342.342128:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766342/real 1775766342] req@ffff8f99703d0380 x1862022967212672/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766342.342145:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0380 x1862022967212672/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766342.342175:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766342.342180:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766342.342185:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967212672:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766342.342207:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967212800:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766342.342230:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766342.342240:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766342/real 1775766342] req@ffff8f99703d0000 x1862022967212800/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766342.342249:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0000 x1862022967212800/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766342.342255:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766342.342257:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766342.342259:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967212800:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766342.342263:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967212928:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766342.342274:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766342.342280:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766342/real 1775766342] req@ffff8f99703d2680 x1862022967212928/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766342.342287:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2680 x1862022967212928/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766342.342292:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766342.342294:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766342.342296:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967212928:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766342.342313:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967213056:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766342.342325:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766342.342331:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766342/real 1775766342] req@ffff8f99703d3800 x1862022967213056/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766342.342339:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3800 x1862022967213056/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766342.342344:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766342.342346:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766342.342348:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967213056:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766343.237862:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766343.237923:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766344.261889:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766344.261967:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766345.286049:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766345.286051:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766346.309838:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766346.309884:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766347.333842:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766347.333906:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766347.461830:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766347.461838:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766347.461849:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766347.461858:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2969 00000100:00080000:2.0:1775766347.461869:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766347.461874:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766347.461911:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3800 x1862022967213184/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766347.461934:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766347.461937:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766347.461938:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766347.461941:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2969 00000100:00080000:2.0:1775766347.461943:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766347.461945:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766347.461953:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766347.461961:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2680 x1862022967213312/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766347.461969:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766347.461971:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766347.461973:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766347.461975:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2969 00000100:00100000:1.0:1775766347.461983:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967213184:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775766347.462012:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766347.462015:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766347.462020:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766347.462028:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0000 x1862022967213440/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766347.462036:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766347.462038:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766347.462039:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766347.462042:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2969 00000100:00080000:2.0:1775766347.462045:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1775766347.462046:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775766347.462050:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766347.462053:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766347.462061:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0380 x1862022967213568/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766347.462089:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766347/real 1775766347] req@ffff8f99703d3800 x1862022967213184/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766347.462104:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3800 x1862022967213184/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766347.462111:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766347.462114:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766347.462117:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967213184:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766347.462123:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967213312:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766347.462157:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766347.462166:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766347/real 1775766347] req@ffff8f99703d2680 x1862022967213312/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766347.462175:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2680 x1862022967213312/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766347.462180:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766347.462183:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766347.462185:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967213312:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766347.462202:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967213440:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766347.462213:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766347.462219:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766347/real 1775766347] req@ffff8f99703d0000 x1862022967213440/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766347.462226:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0000 x1862022967213440/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766347.462231:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766347.462233:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766347.462235:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967213440:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766347.462239:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967213568:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766347.462248:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766347.462253:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766347/real 1775766347] req@ffff8f99703d0380 x1862022967213568/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766347.462276:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0380 x1862022967213568/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766347.462282:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766347.462284:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766347.462286:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967213568:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766348.357882:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766348.357964:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766349.381847:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766349.381908:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766350.405819:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766350.405864:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766351.429831:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766351.429932:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766352.453945:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766352.453948:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00080000:2.0:1775766352.581823:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766352.581831:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766352.581835:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766352.581844:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2974 00000100:00080000:2.0:1775766352.581861:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766352.581867:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766352.581940:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2d80 x1862022967213696/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766352.581969:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766352.581972:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766352.581974:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766352.581978:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2974 00000100:00080000:2.0:1775766352.581982:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766352.581985:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766352.581995:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766352.582008:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0e00 x1862022967213824/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766352.582020:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967213696:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775766352.582020:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766352.582028:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766352.582031:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766352.582036:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2974 00000100:00080000:2.0:1775766352.582041:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766352.582045:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766352.582051:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766352.582064:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0700 x1862022967213952/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766352.582075:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766352.582078:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766352.582081:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1775766352.582085:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775766352.582104:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2974 00000100:00080000:2.0:1775766352.582110:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766352.582115:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766352.582120:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766352.582132:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1500 x1862022967214080/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766352.582138:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766352/real 1775766352] req@ffff8f99703d2d80 x1862022967213696/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766352.582161:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2d80 x1862022967213696/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766352.582181:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766352.582187:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766352.582192:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967213696:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766352.582213:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967213824:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766352.582233:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766352.582254:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766352/real 1775766352] req@ffff8f99703d0e00 x1862022967213824/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766352.582266:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0e00 x1862022967213824/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766352.582294:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766352.582297:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766352.582301:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967213824:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766352.582307:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967213952:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766352.582325:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766352.582334:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766352/real 1775766352] req@ffff8f99703d0700 x1862022967213952/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766352.582347:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0700 x1862022967213952/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766352.582355:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766352.582359:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766352.582370:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967213952:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766352.582376:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967214080:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766352.582392:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766352.582401:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766352/real 1775766352] req@ffff8f99703d1500 x1862022967214080/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766352.582413:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1500 x1862022967214080/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766352.582421:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766352.582424:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766352.582440:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967214080:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766353.477892:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766353.477967:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766354.501816:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766354.501874:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766355.525856:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766355.525918:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766356.549825:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766356.549899:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766357.573896:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766357.573979:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766357.701869:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766357.701880:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766357.701885:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766357.701896:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2979 00000100:00080000:2.0:1775766357.701908:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766357.701914:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766357.701958:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1180 x1862022967214208/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766357.701986:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766357.701988:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766357.701990:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766357.701993:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2979 00000100:00080000:2.0:1775766357.701996:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766357.701998:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766357.702044:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766357.702055:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3100 x1862022967214336/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766357.702074:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766357.702076:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766357.702078:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766357.702082:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2979 00000100:00080000:2.0:1775766357.702085:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766357.702087:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766357.702091:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766357.702099:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967214208:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766357.702100:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3480 x1862022967214464/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766357.702123:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766357.702126:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766357.702127:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766357.702131:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2979 00000100:00080000:2.0:1775766357.702133:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766357.702135:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766357.702141:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766357.702152:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2300 x1862022967214592/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766357.702197:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766357.702273:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766357/real 1775766357] req@ffff8f99703d1180 x1862022967214208/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766357.702302:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1180 x1862022967214208/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766357.702333:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766357.702339:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766357.702344:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967214208:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766357.702353:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967214336:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766357.702373:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766357.702383:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766357/real 1775766357] req@ffff8f99703d3100 x1862022967214336/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766357.702392:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3100 x1862022967214336/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766357.702398:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766357.702400:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766357.702402:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967214336:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766357.702425:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967214464:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766357.702588:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766357.702616:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766357/real 1775766357] req@ffff8f99703d3480 x1862022967214464/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766357.702634:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3480 x1862022967214464/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766357.702660:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766357.702665:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766357.702669:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967214464:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766357.702679:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967214592:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766357.702697:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766357.702708:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766357/real 1775766357] req@ffff8f99703d2300 x1862022967214592/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766357.702720:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2300 x1862022967214592/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766357.702729:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766357.702732:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766357.702736:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967214592:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766358.597851:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766358.597883:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766359.621893:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766359.621905:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766360.645821:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766360.645891:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766361.669822:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766361.669862:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766362.693908:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766362.694005:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766362.821916:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766362.821927:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766362.821932:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766362.821943:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2984 00000100:00080000:0.0:1775766362.821957:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766362.821964:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766362.822029:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967214720/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766362.822084:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766362.822087:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766362.822090:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766362.822093:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2984 00000100:00080000:0.0:1775766362.822096:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766362.822100:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766362.822112:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766362.822146:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967214848/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766362.822146:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967214720:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775766362.822168:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766362.822172:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766362.822175:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766362.822179:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2984 00000100:00080000:0.0:1775766362.822184:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766362.822187:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766362.822193:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766362.822205:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967214976/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766362.822213:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766362.822217:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766362.822218:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766362.822221:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2984 00000100:00080000:0.0:1775766362.822223:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766362.822226:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766362.822229:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775766362.822230:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766362.822245:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967215104/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766362.822314:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766362/real 1775766362] req@ffff8f9970a74700 x1862022967214720/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766362.822468:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967214720/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766362.822479:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766362.822484:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766362.822487:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967214720:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766362.822534:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967214848:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766362.822672:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766362.822704:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766362/real 1775766362] req@ffff8f9970a76d80 x1862022967214848/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766362.822716:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967214848/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766362.822723:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766362.822726:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766362.822729:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967214848:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766362.822735:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967214976:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766362.822829:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766362.822848:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766362/real 1775766362] req@ffff8f9970a74380 x1862022967214976/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766362.822858:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967214976/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766362.822894:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766362.822897:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766362.822900:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967214976:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766362.822905:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967215104:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766362.822920:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766362.822928:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766362/real 1775766362] req@ffff8f9970a75c00 x1862022967215104/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766362.822936:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967215104/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766362.822942:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766362.822945:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766362.822947:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967215104:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766363.717852:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766363.717883:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766364.741829:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766364.741887:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766365.765836:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766365.765874:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766366.789909:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766366.789949:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766367.813827:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766367.813912:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766367.941879:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766367.941887:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766367.941892:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766367.941901:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2989 00000100:00080000:0.0:1775766367.941912:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766367.941919:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766367.941955:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967215232/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766367.941978:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766367.941981:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766367.941982:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766367.941986:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2989 00000100:00080000:0.0:1775766367.941988:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766367.941991:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766367.942002:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766367.942018:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967215232:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766367.942020:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967215360/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766367.942038:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766367.942042:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766367.942107:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1775766367.942114:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775766367.942119:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2989 00000100:00080000:0.0:1775766367.942127:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766367.942133:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766367.942143:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766367.942161:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766367/real 1775766367] req@ffff8f9970a77480 x1862022967215232/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766367.942165:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967215488/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766367.942189:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766367.942192:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1775766367.942194:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967215232/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766367.942195:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766367.942205:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2989 00000100:00080000:0.0:1775766367.942213:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766367.942216:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766367.942218:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766367.942234:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1775766367.942266:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766367.942267:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967215616/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766367.942276:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967215232:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766367.942307:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967215360:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766367.942350:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766367.942362:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766367/real 1775766367] req@ffff8f9970a77800 x1862022967215360/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766367.942373:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967215360/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766367.942379:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766367.942381:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766367.942384:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967215360:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766367.942388:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967215488:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766367.942401:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766367.942407:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766367/real 1775766367] req@ffff8f9970a76a00 x1862022967215488/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766367.942414:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967215488/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766367.942419:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766367.942421:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766367.942423:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967215488:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766367.942427:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967215616:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766367.942452:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766367.942459:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766367/real 1775766367] req@ffff8f9970a75500 x1862022967215616/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766367.942467:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967215616/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766367.942473:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766367.942475:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766367.942477:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967215616:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766368.837872:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766368.837928:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766369.861845:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766369.861895:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766370.885895:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766370.885912:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766371.909806:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766371.909902:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766372.933850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766372.933917:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766373.061846:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766373.061857:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766373.061862:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766373.061873:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2994 00000100:00080000:2.0:1775766373.061886:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766373.061893:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766373.061949:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1c00 x1862022967215744/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766373.061977:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766373.061981:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766373.061984:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766373.061989:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2994 00000100:00080000:2.0:1775766373.061993:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766373.061998:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766373.062011:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766373.062035:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967215744:192.168.202.146@tcp:250:kworker.0 00000800:00000100:1.0:1775766373.062117:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766373.062177:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766373/real 1775766373] req@ffff8f99703d1c00 x1862022967215744/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766373.062183:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3b80 x1862022967215872/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766373.062198:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1c00 x1862022967215744/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766373.062210:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766373.062217:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00080000:2.0:1775766373.062220:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775766373.062225:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967215744:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775766373.062228:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766373.062250:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766373.062258:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2994 00000100:00100000:1.0:1775766373.062268:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967215872:192.168.202.146@tcp:38:kworker.0 00000100:00080000:2.0:1775766373.062270:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766373.062279:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766373.062289:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766373.062324:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1880 x1862022967216000/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766373.062326:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775766373.062342:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766373.062344:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766373.062347:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766373.062351:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2994 00000100:00080000:2.0:1775766373.062354:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775766373.062356:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766373/real 1775766373] req@ffff8f99703d3b80 x1862022967215872/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766373.062361:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766373.062365:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766373.062372:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2a00 x1862022967216128/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766373.062468:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3b80 x1862022967215872/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766373.062481:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766373.062486:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766373.062513:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967215872:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766373.062538:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967216000:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766373.062584:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766373.062603:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766373/real 1775766373] req@ffff8f99703d1880 x1862022967216000/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766373.062611:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1880 x1862022967216000/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766373.062617:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766373.062619:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766373.062622:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967216000:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766373.062626:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967216128:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766373.062637:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766373.062643:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766373/real 1775766373] req@ffff8f99703d2a00 x1862022967216128/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766373.062650:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2a00 x1862022967216128/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766373.062691:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766373.062695:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766373.062699:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967216128:192.168.202.146@tcp:8:kworker.0 00010000:00010000:0.0:1775766373.957885:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766373.957914:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775766374.981853:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766374.981881:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766376.005903:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766376.006001:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766377.029866:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766377.029943:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766378.053828:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766378.053904:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766378.181821:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766378.181832:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766378.181837:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766378.181845:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 2999 00000100:00080000:2.0:1775766378.181855:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766378.181861:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766378.181898:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2a00 x1862022967216256/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766378.181926:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766378.181929:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766378.181931:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766378.181934:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2999 00000100:00080000:2.0:1775766378.181936:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766378.181939:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766378.181947:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766378.181972:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1880 x1862022967216384/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766378.181982:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766378.181986:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766378.181987:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766378.181991:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2999 00000100:00080000:2.0:1775766378.181994:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766378.181997:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766378.182065:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766378.182071:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967216256:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766378.182083:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3b80 x1862022967216512/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766378.182096:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766378.182099:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766378.182101:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766378.182104:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 2999 00000100:00080000:2.0:1775766378.182123:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766378.182126:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766378.182130:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766378.182138:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1c00 x1862022967216640/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766378.182206:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967216384:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766378.182249:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967216512:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766378.182266:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967216640:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766379.077828:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766379.077900:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766380.101860:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766380.101991:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766381.125950:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766381.125969:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000400:00000100:0.0:1775766381.254076:0:2366:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:0.0:1775766381.254128:0:2366:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1775766381.254271:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775766381.254361:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775766381.254373:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775766381.254382:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:1.0:1775766381.254557:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766378/real 1775766381] req@ffff8f99703d3b80 x1862022967216512/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766381.254589:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3b80 x1862022967216512/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766381.254603:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766381.254608:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766381.254614:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967216512:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766381.254622:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766378/real 1775766381] req@ffff8f99703d1c00 x1862022967216640/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766381.254650:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1c00 x1862022967216640/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766381.254681:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766381.254685:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766381.254690:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967216640:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766381.254698:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766378/real 1775766381] req@ffff8f99703d2a00 x1862022967216256/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766381.254715:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2a00 x1862022967216256/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766381.254723:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766381.254727:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766381.254730:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967216256:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766381.254814:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766378/real 1775766381] req@ffff8f99703d1880 x1862022967216384/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766381.254833:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1880 x1862022967216384/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766381.254848:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766381.254852:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766381.254856:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967216384:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766382.149847:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766382.149907:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766383.173838:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766383.173938:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766383.301899:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766383.301909:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766383.301914:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766383.301923:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3004 00000100:00080000:0.0:1775766383.301934:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766383.301942:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766383.301977:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967216768/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766383.302015:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766383.302018:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766383.302028:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766383.302032:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3004 00000100:00080000:0.0:1775766383.302036:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766383.302040:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766383.302050:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766383.302062:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967216896/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766383.302070:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967216768:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775766383.302075:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766383.302080:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766383.302121:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766383.302126:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3004 00000100:00080000:0.0:1775766383.302130:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766383.302133:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766383.302139:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775766383.302140:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766383.302156:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967217024/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766383.302165:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766383.302168:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766383.302169:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766383.302172:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3004 00000100:00080000:0.0:1775766383.302176:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766383.302179:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766383.302183:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766383.302188:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766383/real 1775766383] req@ffff8f9970a77100 x1862022967216768/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766383.302196:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967217152/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766383.302260:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967216768/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766383.302271:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766383.302275:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766383.302279:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967216768:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766383.302303:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967216896:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766383.302327:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766383.302340:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766383/real 1775766383] req@ffff8f9970a76300 x1862022967216896/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766383.302351:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967216896/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766383.302388:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766383.302428:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766383.302435:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967216896:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766383.302444:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967217024:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766383.302476:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766383.302495:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766383/real 1775766383] req@ffff8f9970a75180 x1862022967217024/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766383.302510:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967217024/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766383.302550:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766383.302555:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766383.302560:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967217024:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766383.302568:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967217152:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766383.302597:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766383.302613:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766383/real 1775766383] req@ffff8f9970a75500 x1862022967217152/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766383.302630:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967217152/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766383.302640:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766383.302644:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766383.302648:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967217152:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766384.197873:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766384.197974:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766385.221872:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766385.221954:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766386.245932:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766386.245999:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766387.269849:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766387.269939:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766388.293947:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766388.294002:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766388.421875:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766388.421886:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766388.421892:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766388.421903:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3010 00000100:00080000:0.0:1775766388.421916:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766388.421923:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766388.421978:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967217280/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766388.422004:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766388.422007:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766388.422008:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766388.422012:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3010 00000100:00080000:0.0:1775766388.422015:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766388.422017:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766388.422025:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766388.422045:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967217408/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766388.422053:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766388.422056:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766388.422059:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766388.422065:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3010 00000100:00080000:0.0:1775766388.422068:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766388.422070:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766388.422074:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766388.422082:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967217536/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766388.422091:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766388.422094:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766388.422095:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766388.422102:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3010 00000100:00080000:0.0:1775766388.422111:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766388.422115:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766388.422115:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967217280:192.168.202.146@tcp:250:kworker.0 00000008:00100000:0.0:1775766388.422132:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766388.422141:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967217664/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766388.422245:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766388.422327:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766388/real 1775766388] req@ffff8f9970a76a00 x1862022967217280/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766388.422413:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967217280/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766388.422428:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766388.422633:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766388.422640:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967217280:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766388.422651:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967217408:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766388.422720:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766388.422792:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766388/real 1775766388] req@ffff8f9970a77800 x1862022967217408/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766388.422808:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967217408/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766388.422815:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766388.422818:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766388.422820:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967217408:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766388.422862:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967217536:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766388.422880:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766388.422889:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766388/real 1775766388] req@ffff8f9970a77480 x1862022967217536/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766388.422898:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967217536/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766388.422904:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766388.422906:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766388.422909:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967217536:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766388.422913:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967217664:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766388.422969:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766388.422976:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766388/real 1775766388] req@ffff8f9970a75c00 x1862022967217664/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766388.422985:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967217664/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766388.422990:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766388.422992:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766388.422994:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967217664:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766389.317822:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766389.317941:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766390.341879:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766390.342250:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766391.365913:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766391.366073:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766392.389887:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766392.389976:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766393.413906:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766393.413992:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766393.541905:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766393.541918:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766393.541923:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766393.541935:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3015 00000100:00080000:0.0:1775766393.541950:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766393.541959:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766393.542017:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967217792/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766393.542056:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766393.542063:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766393.542067:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766393.542076:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3015 00000100:00080000:0.0:1775766393.542083:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766393.542090:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766393.542106:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766393.542148:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967217792:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766393.542185:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967217920/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766393.542222:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766393.542227:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766393.542230:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766393.542236:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3015 00000100:00080000:0.0:1775766393.542253:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1775766393.542258:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775766393.542259:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766393.542268:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766393.542285:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967218048/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766393.542348:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766393/real 1775766393] req@ffff8f9970a74380 x1862022967217792/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766393.542380:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967217792/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766393.542381:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766393.542388:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766393.542391:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766393.542397:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3015 00000100:00080000:1.0:1775766393.542398:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766393.542404:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766393.542405:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00080000:0.0:1775766393.542411:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766393.542412:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967217792:192.168.202.146@tcp:250:kworker.0 00000008:00100000:0.0:1775766393.542420:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766393.542439:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967218176/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766393.542444:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967217920:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766393.542485:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766393.542509:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766393/real 1775766393] req@ffff8f9970a76d80 x1862022967217920/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766393.542564:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967217920/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766393.542575:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766393.542579:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766393.542583:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967217920:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766393.542590:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967218048:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766393.542618:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766393.542632:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766393/real 1775766393] req@ffff8f9970a74700 x1862022967218048/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766393.542645:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967218048/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766393.542653:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766393.542656:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766393.542660:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967218048:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766393.542678:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967218176:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766393.542696:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766393.542706:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766393/real 1775766393] req@ffff8f9970a75c00 x1862022967218176/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766393.542735:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967218176/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766393.542818:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766393.542822:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766393.542827:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967218176:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766394.437850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766394.438009:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766395.461986:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766395.462030:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766396.485950:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766396.486006:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766397.509892:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766397.509991:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766398.533886:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766398.533900:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766398.661863:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766398.661876:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766398.661941:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766398.661955:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3020 00000100:00080000:0.0:1775766398.661970:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766398.661980:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766398.662029:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967218304/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766398.662066:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766398.662071:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766398.662074:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766398.662079:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3020 00000100:00080000:0.0:1775766398.662083:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766398.662087:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766398.662099:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766398.662114:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967218432/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766398.662129:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766398.662133:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766398.662136:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766398.662141:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3020 00000100:00080000:0.0:1775766398.662144:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766398.662148:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766398.662155:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766398.662171:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967218560/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766398.662173:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967218304:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775766398.662208:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766398.662212:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766398.662215:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766398.662220:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3020 00000100:00080000:0.0:1775766398.662224:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766398.662228:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766398.662234:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766398.662274:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967218688/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766398.662314:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766398.662383:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766398/real 1775766398] req@ffff8f9970a77480 x1862022967218304/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766398.662424:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967218304/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766398.662439:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766398.662444:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766398.662450:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967218304:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766398.662460:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967218432:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766398.662490:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766398.662505:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766398/real 1775766398] req@ffff8f9970a77800 x1862022967218432/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766398.662520:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967218432/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766398.662570:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766398.662575:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766398.662580:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967218432:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766398.662611:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967218560:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766398.662634:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766398.662648:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766398/real 1775766398] req@ffff8f9970a76a00 x1862022967218560/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766398.662662:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967218560/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766398.662672:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766398.662676:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766398.662680:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967218560:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766398.662686:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967218688:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766398.662703:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766398.662714:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766398/real 1775766398] req@ffff8f9970a75500 x1862022967218688/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766398.662727:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967218688/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766398.662770:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766398.662775:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766398.662800:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967218688:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766399.557887:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766399.557962:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766400.581859:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766400.582009:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766401.605878:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766401.606057:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766402.629897:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766402.629919:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766403.653857:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766403.653984:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766403.782772:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766403.782787:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766403.782794:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766403.782810:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3025 00000100:00080000:2.0:1775766403.782827:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766403.782836:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766403.782898:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2300 x1862022967218816/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766403.782940:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766403.782945:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766403.782948:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766403.782955:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3025 00000100:00080000:2.0:1775766403.782960:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766403.782965:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766403.782979:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766403.783003:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3480 x1862022967218944/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766403.783025:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967218816:192.168.202.146@tcp:250:kworker.0 00000800:00000100:3.0:1775766403.783121:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775766403.783166:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766403.783173:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1775766403.783175:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766403/real 1775766403] req@ffff8f99703d2300 x1862022967218816/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766403.783179:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766403.783213:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3025 00000100:00080000:2.0:1775766403.783227:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766403.783236:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766403.783249:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766403.783283:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3100 x1862022967219072/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766403.783294:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2300 x1862022967218816/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766403.783323:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766403.783327:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766403.783328:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766403.783336:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766403.783342:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00080000:2.0:1775766403.783349:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3025 00000100:00080000:2.0:1775766403.783353:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766403.783357:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766403.783361:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766403.783369:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1180 x1862022967219200/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766403.783522:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967218816:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766403.783574:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967218944:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766403.783686:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766403.783706:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766403/real 1775766403] req@ffff8f99703d3480 x1862022967218944/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766403.783724:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3480 x1862022967218944/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766403.783736:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766403.783786:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766403.783792:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967218944:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766403.783801:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967219072:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766403.783829:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766403.783842:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766403/real 1775766403] req@ffff8f99703d3100 x1862022967219072/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766403.783852:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3100 x1862022967219072/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766403.783859:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766403.783862:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766403.783864:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967219072:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766403.783870:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967219200:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766403.783885:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766403.783892:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766403/real 1775766403] req@ffff8f99703d1180 x1862022967219200/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766403.783900:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1180 x1862022967219200/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766403.783912:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766403.783915:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766403.783917:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967219200:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766404.677860:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766404.677930:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766405.701917:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766405.702045:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766406.725815:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766406.725955:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766407.749885:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766407.749981:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766408.773833:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766408.773934:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766408.901853:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766408.901860:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766408.901864:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766408.901883:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3030 00000100:00080000:2.0:1775766408.901892:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766408.901898:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766408.901934:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1500 x1862022967219328/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766408.901967:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766408.901970:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766408.901971:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766408.901974:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3030 00000100:00080000:2.0:1775766408.901977:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766408.901979:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766408.901987:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766408.901999:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0700 x1862022967219456/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766408.902000:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967219328:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775766408.902013:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766408.902016:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766408.902017:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766408.902019:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3030 00000100:00080000:2.0:1775766408.902021:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766408.902024:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766408.902027:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766408.902064:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766408.902075:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0e00 x1862022967219584/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766408.902083:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766408.902085:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766408.902086:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766408.902089:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3030 00000100:00080000:2.0:1775766408.902108:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766408.902111:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766408.902116:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766408.902127:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2d80 x1862022967219712/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766408.902163:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766408/real 1775766408] req@ffff8f99703d1500 x1862022967219328/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766408.902211:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1500 x1862022967219328/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766408.902223:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766408.902228:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766408.902231:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967219328:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766408.902251:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967219456:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766408.902269:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766408.902279:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766408/real 1775766408] req@ffff8f99703d0700 x1862022967219456/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766408.902287:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0700 x1862022967219456/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766408.902292:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766408.902294:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766408.902297:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967219456:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766408.902301:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967219584:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766408.902312:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766408.902318:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766408/real 1775766408] req@ffff8f99703d0e00 x1862022967219584/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766408.902325:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0e00 x1862022967219584/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766408.902330:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766408.902332:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766408.902333:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967219584:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766408.902337:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967219712:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766408.902346:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766408.902351:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766408/real 1775766408] req@ffff8f99703d2d80 x1862022967219712/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766408.902358:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2d80 x1862022967219712/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766408.902363:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766408.902364:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766408.902366:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967219712:192.168.202.146@tcp:8:kworker.0 00010000:00010000:0.0:1775766409.797929:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766409.797943:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775766410.821882:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766410.821947:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766411.845867:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766411.846009:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766412.869889:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766412.869930:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775766413.893939:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766413.893969:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766414.021919:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766414.021930:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766414.021935:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766414.021945:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3035 00000100:00080000:1.0:1775766414.021957:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766414.021964:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766414.022003:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967219840/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766414.022034:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766414.022038:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766414.022040:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766414.022044:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3035 00000100:00080000:1.0:1775766414.022047:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766414.022051:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766414.022060:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766414.022073:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967219968/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766414.022079:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967219840:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775766414.022141:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766414.022144:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766414.022146:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766414.022152:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3035 00000100:00080000:1.0:1775766414.022157:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766414.022161:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:3.0:1775766414.022167:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:1.0:1775766414.022169:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766414.022186:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967220096/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766414.022196:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766414.022199:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766414.022201:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766414.022214:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3035 00000100:00080000:1.0:1775766414.022218:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766414.022222:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766414.022227:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766414/real 1775766414] req@ffff8f994f2e6680 x1862022967219840/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1775766414.022230:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766414.022250:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967220224/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766414.022257:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967219840/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766414.022286:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766414.022292:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766414.022297:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967219840:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766414.022322:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967219968:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766414.022344:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766414.022355:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766414/real 1775766414] req@ffff8f994f2e5f80 x1862022967219968/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766414.022366:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967219968/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766414.022373:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766414.022376:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766414.022379:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967219968:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766414.022384:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967220096:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766414.022397:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766414.022404:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766414/real 1775766414] req@ffff8f994f2e6a00 x1862022967220096/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766414.022426:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967220096/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766414.022439:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766414.022442:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766414.022445:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967220096:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766414.022450:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967220224:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766414.022464:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766414.022472:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766414/real 1775766414] req@ffff8f994f2e4000 x1862022967220224/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766414.022481:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967220224/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766414.022489:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766414.022491:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766414.022493:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967220224:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766414.917909:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766414.917987:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766415.941902:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766415.941943:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766416.965903:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766416.965904:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766417.989889:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766417.990021:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766419.013902:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766419.013909:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766419.141896:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766419.141908:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766419.141913:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766419.141939:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3040 00000100:00080000:0.0:1775766419.141966:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766419.141977:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766419.142063:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967220352/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766419.142107:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766419.142111:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766419.142113:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766419.142118:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3040 00000100:00080000:0.0:1775766419.142122:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766419.142127:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766419.142138:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766419.142149:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967220352:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766419.142155:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967220480/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766419.142165:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766419.142169:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766419.142170:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766419.142174:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3040 00000100:00080000:0.0:1775766419.142177:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766419.142180:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766419.142183:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766419.142194:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967220608/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766419.142202:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766419.142205:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766419.142206:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766419.142209:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3040 00000100:00080000:0.0:1775766419.142213:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766419.142216:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766419.142220:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766419.142244:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967220736/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766419.142248:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766419.142317:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766419/real 1775766419] req@ffff8f9970a75180 x1862022967220352/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766419.142348:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967220352/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766419.142362:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766419.142369:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766419.142373:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967220352:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766419.142398:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967220480:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766419.142421:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766419.142432:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766419/real 1775766419] req@ffff8f9970a76300 x1862022967220480/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766419.142442:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967220480/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766419.142449:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766419.142452:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766419.142454:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967220480:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766419.142459:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967220608:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766419.142473:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766419.142479:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766419/real 1775766419] req@ffff8f9970a77100 x1862022967220608/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766419.142488:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967220608/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766419.142494:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766419.142496:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766419.142498:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967220608:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766419.142503:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967220736:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766419.142515:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766419.142521:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766419/real 1775766419] req@ffff8f9970a75500 x1862022967220736/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766419.142530:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967220736/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766419.142536:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766419.142538:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766419.142540:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967220736:192.168.202.146@tcp:8:kworker.0 00010000:00010000:0.0:1775766420.037908:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766420.037924:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775766421.061802:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766421.061863:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766422.085868:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766422.085978:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766423.109926:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766423.110018:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766424.133893:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766424.133917:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00080000:2.0:1775766424.261883:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766424.261931:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766424.261935:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766424.261945:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3045 00000100:00080000:2.0:1775766424.261957:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766424.261963:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766424.262027:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0380 x1862022967220864/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766424.262053:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766424.262056:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766424.262058:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766424.262062:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3045 00000100:00080000:2.0:1775766424.262066:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766424.262070:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766424.262080:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766424.262091:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967220864:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766424.262099:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0000 x1862022967220992/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766424.262115:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766424.262120:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766424.262122:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766424.262127:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3045 00000100:00080000:2.0:1775766424.262133:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766424.262138:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766424.262146:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766424.262162:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766424.262164:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2680 x1862022967221120/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766424.262193:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766424.262196:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766424.262198:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766424.262202:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3045 00000100:00080000:2.0:1775766424.262206:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766424.262210:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766424.262216:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766424.262224:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766424/real 1775766424] req@ffff8f99703d0380 x1862022967220864/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766424.262230:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3800 x1862022967221248/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766424.262264:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0380 x1862022967220864/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766424.262287:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766424.262291:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766424.262294:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967220864:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766424.262316:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967220992:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766424.262336:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766424.262346:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766424/real 1775766424] req@ffff8f99703d0000 x1862022967220992/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766424.262355:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0000 x1862022967220992/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766424.262367:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766424.262370:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766424.262372:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967220992:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766424.262377:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967221120:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766424.262390:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766424.262396:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766424/real 1775766424] req@ffff8f99703d2680 x1862022967221120/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766424.262405:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2680 x1862022967221120/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766424.262411:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766424.262413:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766424.262415:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967221120:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766424.262419:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967221248:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766424.262431:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766424.262437:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766424/real 1775766424] req@ffff8f99703d3800 x1862022967221248/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766424.262446:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3800 x1862022967221248/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766424.262451:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766424.262454:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766424.262457:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967221248:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766425.157858:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766425.157919:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766426.181806:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766426.181880:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766427.205841:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766427.205916:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766428.229878:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766428.229928:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766429.253845:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766429.253939:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766429.381831:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766429.381838:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766429.381842:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766429.381850:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3050 00000100:00080000:0.0:1775766429.381860:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766429.381865:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766429.381896:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967221376/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766429.381944:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766429.381947:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766429.381949:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766429.381953:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3050 00000100:00080000:0.0:1775766429.381957:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766429.381960:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766429.381968:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967221376:192.168.202.146@tcp:250:kworker.0 00000008:00100000:0.0:1775766429.381969:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766429.381984:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967221504/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766429.382022:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766429.382024:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766429.382026:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766429.382029:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3050 00000800:00000100:3.0:1775766429.382033:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775766429.382035:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766429.382039:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766429.382044:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766429.382053:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967221632/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766429.382060:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766429.382063:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766429.382064:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766429.382068:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3050 00000100:00080000:0.0:1775766429.382070:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766429.382072:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766429.382074:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766429.382081:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967221760/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766429.382086:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766429/real 1775766429] req@ffff8f9970a76a00 x1862022967221376/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766429.382106:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967221376/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766429.382115:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766429.382118:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766429.382121:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967221376:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766429.382142:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967221504:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766429.382159:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766429.382167:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766429/real 1775766429] req@ffff8f9970a77800 x1862022967221504/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766429.382174:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967221504/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766429.382179:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766429.382181:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766429.382183:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967221504:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766429.382187:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967221632:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766429.382197:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766429.382204:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766429/real 1775766429] req@ffff8f9970a77480 x1862022967221632/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766429.382211:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967221632/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766429.382216:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766429.382218:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766429.382220:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967221632:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766429.382223:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967221760:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766429.382233:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766429.382238:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766429/real 1775766429] req@ffff8f9970a75c00 x1862022967221760/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766429.382245:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967221760/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766429.382251:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766429.382252:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766429.382254:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967221760:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766430.277811:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766430.277909:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766431.301862:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766431.301876:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766432.325924:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766432.325964:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775766433.349850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766433.349944:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766434.373825:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766434.373905:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766434.501848:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766434.501856:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766434.501860:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766434.501889:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3056 00000100:00080000:2.0:1775766434.501899:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766434.501905:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766434.501972:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3800 x1862022967221888/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766434.501994:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766434.501997:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766434.501998:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766434.502001:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3056 00000100:00080000:2.0:1775766434.502004:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766434.502007:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766434.502015:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766434.502023:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2680 x1862022967222016/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766434.502031:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775766434.502033:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967221888:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775766434.502034:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766434.502038:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766434.502044:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3056 00000100:00080000:2.0:1775766434.502046:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766434.502048:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766434.502051:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766434.502059:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0000 x1862022967222144/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766434.502065:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766434.502067:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766434.502069:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766434.502086:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3056 00000100:00080000:2.0:1775766434.502092:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766434.502096:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766434.502102:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766434.502105:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766434.502120:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0380 x1862022967222272/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766434.502154:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766434/real 1775766434] req@ffff8f99703d3800 x1862022967221888/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766434.502170:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3800 x1862022967221888/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766434.502183:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766434.502188:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766434.502193:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967221888:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766434.502218:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967222016:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766434.502248:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766434.502259:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766434/real 1775766434] req@ffff8f99703d2680 x1862022967222016/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766434.502272:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2680 x1862022967222016/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766434.502280:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766434.502284:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766434.502287:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967222016:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766434.502294:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967222144:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766434.502324:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766434.502334:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766434/real 1775766434] req@ffff8f99703d0000 x1862022967222144/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766434.502342:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0000 x1862022967222144/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766434.502348:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766434.502350:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766434.502352:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967222144:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766434.502356:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967222272:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766434.502369:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766434.502375:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766434/real 1775766434] req@ffff8f99703d0380 x1862022967222272/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766434.502382:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0380 x1862022967222272/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766434.502387:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766434.502389:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766434.502391:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967222272:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766435.397873:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766435.397949:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766436.421843:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766436.421924:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766437.445904:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766437.445980:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766438.469917:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766438.469949:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:1.0:1775766439.493832:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766439.493891:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766439.621830:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766439.621839:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766439.621843:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766439.621852:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3061 00000100:00080000:2.0:1775766439.621862:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766439.621868:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766439.621902:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2d80 x1862022967222400/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766439.621926:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766439.621929:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766439.621931:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766439.621934:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3061 00000100:00080000:2.0:1775766439.621937:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766439.621939:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766439.621947:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766439.621988:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967222400:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766439.621989:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0e00 x1862022967222528/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766439.622004:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766439.622006:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766439.622008:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766439.622011:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3061 00000100:00080000:2.0:1775766439.622014:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766439.622017:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766439.622022:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766439.622033:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0700 x1862022967222656/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766439.622041:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766439.622043:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766439.622045:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766439.622047:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3061 00000100:00080000:2.0:1775766439.622050:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1775766439.622055:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775766439.622057:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766439.622062:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766439.622072:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1500 x1862022967222784/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766439.622096:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766439/real 1775766439] req@ffff8f99703d2d80 x1862022967222400/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766439.622112:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2d80 x1862022967222400/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766439.622121:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766439.622125:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766439.622129:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967222400:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766439.622152:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967222528:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766439.622167:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766439.622175:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766439/real 1775766439] req@ffff8f99703d0e00 x1862022967222528/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766439.622206:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0e00 x1862022967222528/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766439.622211:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766439.622213:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766439.622216:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967222528:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766439.622220:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967222656:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766439.622233:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766439.622239:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766439/real 1775766439] req@ffff8f99703d0700 x1862022967222656/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766439.622247:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0700 x1862022967222656/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766439.622252:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766439.622254:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766439.622256:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967222656:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766439.622260:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967222784:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766439.622270:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766439.622276:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766439/real 1775766439] req@ffff8f99703d1500 x1862022967222784/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766439.622283:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1500 x1862022967222784/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766439.622303:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766439.622306:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766439.622309:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967222784:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766440.517912:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766440.518000:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766441.541830:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766441.541907:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766442.565875:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766442.565947:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766443.589876:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766443.589937:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766444.613856:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766444.613926:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766444.741867:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766444.741877:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766444.741882:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766444.741940:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3066 00000100:00080000:2.0:1775766444.741953:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766444.741961:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766444.742006:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1180 x1862022967222912/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766444.742031:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766444.742050:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766444.742052:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766444.742056:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3066 00000100:00080000:2.0:1775766444.742059:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766444.742062:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766444.742071:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766444.742083:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3100 x1862022967223040/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766444.742092:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766444.742095:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766444.742096:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766444.742099:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3066 00000100:00080000:2.0:1775766444.742102:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775766444.742106:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967222912:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775766444.742108:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766444.742135:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766444.742154:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3480 x1862022967223168/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766444.742187:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766444.742190:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766444.742191:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766444.742194:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3066 00000100:00080000:2.0:1775766444.742199:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766444.742202:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766444.742206:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766444.742215:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2300 x1862022967223296/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766444.742260:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967223040:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766444.742315:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967223168:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766444.742358:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967223296:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766445.637834:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766445.637893:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766446.661831:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766446.661903:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766447.685809:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766447.685865:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775766447.814034:0:2368:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:2.0:1775766447.814069:0:2368:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775766447.814083:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:2.0:1775766447.814115:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:2.0:1775766447.814121:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:2.0:1775766447.814129:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:1.0:1775766447.814152:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766444/real 1775766447] req@ffff8f99703d3480 x1862022967223168/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766447.814182:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3480 x1862022967223168/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766447.814195:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766447.814201:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766447.814206:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967223168:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766447.814240:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766444/real 1775766447] req@ffff8f99703d2300 x1862022967223296/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766447.814250:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2300 x1862022967223296/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766447.814257:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766447.814261:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766447.814264:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967223296:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766447.814271:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766444/real 1775766447] req@ffff8f99703d1180 x1862022967222912/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766447.814283:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1180 x1862022967222912/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766447.814291:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766447.814295:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766447.814298:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967222912:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766447.814306:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766444/real 1775766447] req@ffff8f99703d3100 x1862022967223040/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766447.814316:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3100 x1862022967223040/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766447.814324:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766447.814327:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766447.814331:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967223040:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766448.709870:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766448.709944:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766449.733830:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766449.733904:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766449.861883:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766449.861895:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766449.861901:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766449.861914:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3071 00000100:00080000:1.0:1775766449.861929:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766449.861938:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766449.861979:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967223424/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766449.862016:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766449.862021:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766449.862023:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766449.862029:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3071 00000100:00080000:1.0:1775766449.862034:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766449.862040:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766449.862063:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766449.862066:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967223424:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766449.862084:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967223552/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766449.862101:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766449.862105:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766449.862107:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766449.862113:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3071 00000100:00080000:1.0:1775766449.862118:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1775766449.862144:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775766449.862177:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766449.862185:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766449.862202:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967223680/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766449.862214:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766449.862219:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766449.862221:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766449.862226:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3071 00000100:00080000:1.0:1775766449.862231:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775766449.862236:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766449/real 1775766449] req@ffff8f994f2e5180 x1862022967223424/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766449.862241:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766449.862262:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766449.862281:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967223808/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766449.862286:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967223424/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766449.862322:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766449.862329:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766449.862336:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967223424:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766449.862377:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967223552:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766449.862414:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766449.862432:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766449/real 1775766449] req@ffff8f994f2e7800 x1862022967223552/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766449.862451:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967223552/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766449.862463:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766449.862468:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766449.862472:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967223552:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766449.862480:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967223680:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766449.862510:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766449.862525:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766449/real 1775766449] req@ffff8f994f2e4a80 x1862022967223680/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766449.862541:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967223680/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766449.862551:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766449.862555:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766449.862560:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967223680:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766449.862568:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967223808:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766449.862594:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766449.862609:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766449/real 1775766449] req@ffff8f994f2e6d80 x1862022967223808/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766449.862623:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967223808/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766449.862661:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766449.862666:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766449.862670:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967223808:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766450.757839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766450.757891:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766451.781818:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766451.781857:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766452.805815:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766452.805907:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766453.829849:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766453.829883:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766454.853908:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766454.853916:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766454.981865:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766454.981877:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766454.981883:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766454.981896:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3076 00000100:00080000:1.0:1775766454.981909:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766454.981916:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766454.981967:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967223936/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766454.982113:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766454.982117:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766454.982120:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766454.982128:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3076 00000100:00080000:1.0:1775766454.982139:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766454.982146:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766454.982154:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967223936:192.168.202.146@tcp:250:kworker.0 00000008:00100000:1.0:1775766454.982157:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766454.982190:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967224064/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766454.982200:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766454.982203:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766454.982204:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766454.982207:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3076 00000100:00080000:1.0:1775766454.982210:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766454.982213:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766454.982216:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766454.982226:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967224192/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766454.982233:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775766454.982242:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766454.982245:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766454.982247:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766454.982252:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3076 00000100:00080000:1.0:1775766454.982277:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766454.982281:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766454.982285:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766454.982290:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766454/real 1775766454] req@ffff8f994f2e6d80 x1862022967223936/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766454.982301:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967224320/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766454.982311:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967223936/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766454.982325:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766454.982335:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766454.982341:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967223936:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766454.982374:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967224064:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766454.982404:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766454.982419:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766454/real 1775766454] req@ffff8f994f2e4a80 x1862022967224064/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766454.982430:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967224064/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766454.982436:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766454.982439:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766454.982442:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967224064:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766454.982447:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967224192:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766454.982463:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766454.982470:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766454/real 1775766454] req@ffff8f994f2e7800 x1862022967224192/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766454.982479:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967224192/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766454.982485:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766454.982487:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766454.982489:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967224192:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766454.982493:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967224320:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766454.982504:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766454.982510:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766454/real 1775766454] req@ffff8f994f2e5180 x1862022967224320/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766454.982518:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967224320/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766454.982524:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766454.982526:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766454.982528:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967224320:192.168.202.146@tcp:8:kworker.0 00010000:00010000:0.0:1775766455.877878:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766455.877882:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:1.0:1775766456.901847:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766456.901912:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766457.925830:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766457.925865:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766458.949826:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766458.949910:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766459.973820:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766459.973896:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766460.101879:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766460.101887:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766460.101891:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766460.101898:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3081 00000100:00080000:2.0:1775766460.101908:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766460.101950:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766460.101995:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1880 x1862022967224448/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766460.102026:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766460.102029:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766460.102030:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766460.102039:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3081 00000100:00080000:2.0:1775766460.102042:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766460.102045:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766460.102054:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766460.102071:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2a00 x1862022967224576/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766460.102072:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967224448:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775766460.102086:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766460.102089:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766460.102090:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766460.102093:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3081 00000100:00080000:2.0:1775766460.102095:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766460.102097:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766460.102100:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766460.102107:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1c00 x1862022967224704/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766460.102113:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766460.102115:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766460.102116:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766460.102118:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3081 00000100:00080000:2.0:1775766460.102127:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766460.102130:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766460.102133:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766460.102147:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3b80 x1862022967224832/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766460.102166:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766460.102220:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766460/real 1775766460] req@ffff8f99703d1880 x1862022967224448/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766460.102242:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1880 x1862022967224448/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766460.102253:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766460.102257:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766460.102261:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967224448:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766460.102285:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967224576:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766460.102307:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766460.102320:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766460/real 1775766460] req@ffff8f99703d2a00 x1862022967224576/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766460.102333:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2a00 x1862022967224576/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766460.102342:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766460.102345:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766460.102348:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967224576:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766460.102354:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967224704:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766460.102373:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766460.102383:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766460/real 1775766460] req@ffff8f99703d1c00 x1862022967224704/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766460.102396:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1c00 x1862022967224704/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766460.102411:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766460.102415:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766460.102418:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967224704:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766460.102424:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967224832:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766460.102443:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766460.102453:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766460/real 1775766460] req@ffff8f99703d3b80 x1862022967224832/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766460.102463:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3b80 x1862022967224832/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766460.102471:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766460.102474:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766460.102476:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967224832:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766460.997863:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766460.997932:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766462.021865:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766462.021967:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766463.045836:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766463.046006:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766464.069880:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766464.069958:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766465.093816:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766465.093881:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766465.221838:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766465.221888:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766465.221894:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766465.221904:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3086 00000100:00080000:2.0:1775766465.221916:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766465.221923:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766465.221962:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3b80 x1862022967224960/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766465.221987:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766465.221990:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766465.221992:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766465.221995:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3086 00000100:00080000:2.0:1775766465.221999:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766465.222004:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766465.222013:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766465.222028:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1c00 x1862022967225088/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766465.222037:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967224960:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775766465.222044:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766465.222064:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766465.222066:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766465.222069:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3086 00000100:00080000:2.0:1775766465.222073:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766465.222076:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766465.222080:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766465.222099:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2a00 x1862022967225216/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766465.222106:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766465.222109:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766465.222110:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766465.222113:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3086 00000100:00080000:2.0:1775766465.222122:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766465.222124:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766465.222128:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766465.222135:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1880 x1862022967225344/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766465.222148:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766465.222213:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766465/real 1775766465] req@ffff8f99703d3b80 x1862022967224960/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766465.222238:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3b80 x1862022967224960/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766465.222249:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766465.222255:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766465.222259:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967224960:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766465.222284:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967225088:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766465.222307:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766465.222320:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766465/real 1775766465] req@ffff8f99703d1c00 x1862022967225088/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766465.222343:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1c00 x1862022967225088/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766465.222353:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766465.222356:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766465.222360:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967225088:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766465.222367:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967225216:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766465.222387:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766465.222398:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766465/real 1775766465] req@ffff8f99703d2a00 x1862022967225216/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766465.222411:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2a00 x1862022967225216/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766465.222418:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766465.222422:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766465.222425:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967225216:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766465.222432:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967225344:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766465.222449:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766465.222459:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766465/real 1775766465] req@ffff8f99703d1880 x1862022967225344/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766465.222472:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1880 x1862022967225344/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766465.222480:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766465.222483:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766465.222486:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967225344:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766466.117846:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766466.117933:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766467.141829:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766467.141887:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766468.165862:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766468.165976:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766469.189836:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766469.189912:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766470.213830:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766470.213891:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766470.342996:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766470.343007:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766470.343012:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766470.343024:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3091 00000100:00080000:2.0:1775766470.343035:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766470.343042:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766470.343091:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3100 x1862022967225472/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766470.343155:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766470.343159:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766470.343161:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766470.343166:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3091 00000100:00080000:2.0:1775766470.343171:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766470.343176:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766470.343186:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766470.343190:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967225472:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766470.343204:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1180 x1862022967225600/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766470.343217:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766470.343221:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766470.343223:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766470.343228:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3091 00000100:00080000:2.0:1775766470.343246:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766470.343251:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766470.343257:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766470.343258:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766470.343275:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2300 x1862022967225728/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766470.343287:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766470.343290:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766470.343292:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766470.343298:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3091 00000100:00080000:2.0:1775766470.343317:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766470.343324:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766470.343326:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766470/real 1775766470] req@ffff8f99703d3100 x1862022967225472/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1775766470.343330:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766470.343348:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3480 x1862022967225856/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766470.343358:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3100 x1862022967225472/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766470.343380:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766470.343385:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766470.343391:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967225472:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766470.343416:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967225600:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766470.343442:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766470.343456:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766470/real 1775766470] req@ffff8f99703d1180 x1862022967225600/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766470.343482:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1180 x1862022967225600/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766470.343528:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766470.343532:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766470.343536:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967225600:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766470.343543:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967225728:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766470.343567:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766470.343579:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766470/real 1775766470] req@ffff8f99703d2300 x1862022967225728/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766470.343603:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2300 x1862022967225728/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766470.343610:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766470.343613:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766470.343617:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967225728:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766470.343624:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967225856:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766470.343645:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766470.343656:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766470/real 1775766470] req@ffff8f99703d3480 x1862022967225856/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766470.343680:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3480 x1862022967225856/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766470.343687:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766470.343690:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766470.343693:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967225856:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766471.237818:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766471.237875:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766472.261842:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766472.261960:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766473.285812:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766473.285874:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766474.309814:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766474.309866:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766475.333855:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766475.333919:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766475.461880:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766475.461891:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766475.461896:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766475.461906:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3097 00000100:00080000:3.0:1775766475.461927:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766475.461935:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766475.462016:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf100 x1862022967225984/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766475.462047:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766475.462051:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766475.462053:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766475.462057:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3097 00000100:00080000:3.0:1775766475.462061:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766475.462065:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766475.462075:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766475.462097:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967225984:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766475.462106:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddc00 x1862022967226112/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766475.462130:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766475.462134:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766475.462135:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766475.462140:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3097 00000100:00080000:3.0:1775766475.462144:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766475.462149:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766475.462154:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766475.462179:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967226240/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766475.462190:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766475.462194:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766475.462196:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766475.462201:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3097 00000100:00080000:3.0:1775766475.462206:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766475.462209:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:2.0:1775766475.462216:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:3.0:1775766475.462216:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766475.462242:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967226368/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766475.462278:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766475/real 1775766475] req@ffff8f9970edf100 x1862022967225984/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766475.462299:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf100 x1862022967225984/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766475.462309:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766475.462313:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766475.462317:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967225984:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766475.462358:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967226112:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766475.462392:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766475.462423:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766475/real 1775766475] req@ffff8f9970eddc00 x1862022967226112/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766475.462435:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddc00 x1862022967226112/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766475.462485:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766475.462498:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766475.462516:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967226112:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766475.462523:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967226240:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766475.462545:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766475.462555:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766475/real 1775766475] req@ffff8f9970edce00 x1862022967226240/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766475.462566:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967226240/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766475.462591:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766475.462594:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766475.462597:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967226240:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766475.462602:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967226368:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766475.462618:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766475.462626:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766475/real 1775766475] req@ffff8f9970eded80 x1862022967226368/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766475.462637:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967226368/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766475.462644:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766475.462646:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766475.462649:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967226368:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766476.357846:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766476.357918:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766477.381821:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766477.381899:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766478.405848:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766478.405980:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766479.429813:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766479.429887:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766480.453864:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766480.453911:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766480.581853:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766480.581864:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766480.581870:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766480.581882:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3102 00000100:00080000:3.0:1775766480.581895:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766480.581903:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766480.581942:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967226496/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766480.581975:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766480.581979:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766480.581982:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766480.581988:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3102 00000100:00080000:3.0:1775766480.581991:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766480.581995:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766480.582006:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766480.582021:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967226624/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766480.582034:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775766480.582039:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967226496:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775766480.582040:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766480.582044:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766480.582055:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3102 00000100:00080000:3.0:1775766480.582061:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766480.582068:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766480.582077:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766480.582103:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967226752/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766480.582121:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1775766480.582131:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775766480.582131:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766480.582137:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766480.582146:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3102 00000100:00080000:3.0:1775766480.582153:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766480.582159:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766480.582167:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766480.582188:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967226880/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766480.582248:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766480/real 1775766480] req@ffff8f9970edd500 x1862022967226496/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766480.582275:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967226496/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766480.582294:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766480.582303:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766480.582324:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967226496:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766480.582357:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967226624:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766480.582410:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766480.582431:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766480/real 1775766480] req@ffff8f9970edc700 x1862022967226624/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766480.582446:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967226624/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766480.582456:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766480.582459:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766480.582463:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967226624:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766480.582470:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967226752:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766480.582503:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766480.582513:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766480/real 1775766480] req@ffff8f9970ede680 x1862022967226752/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766480.582525:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967226752/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766480.582583:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766480.582587:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766480.582591:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967226752:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766480.582597:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967226880:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766480.582618:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766480.582629:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766480/real 1775766480] req@ffff8f9970edc000 x1862022967226880/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766480.582641:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967226880/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766480.582650:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766480.582654:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766480.582657:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967226880:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766481.477855:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766481.477996:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766482.501888:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766482.501971:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766483.525860:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766483.525922:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766484.549836:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766484.549919:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766485.573844:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766485.573912:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766485.701861:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766485.701872:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766485.701877:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766485.701887:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3107 00000100:00080000:1.0:1775766485.701900:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766485.701907:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766485.701992:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967227008/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766485.702023:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766485.702033:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766485.702036:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766485.702042:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3107 00000100:00080000:1.0:1775766485.702048:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766485.702070:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766485.702080:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766485.702081:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967227008:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766485.702123:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967227136/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766485.702133:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766485.702136:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766485.702138:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1775766485.702144:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775766485.702152:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3107 00000100:00080000:1.0:1775766485.702157:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766485.702161:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766485.702166:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766485.702183:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766485/real 1775766485] req@ffff8f994f2e4000 x1862022967227008/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766485.702197:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967227008/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766485.702205:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766485.702209:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766485.702212:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967227008:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766485.702231:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967227264/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766485.702245:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766485.702248:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766485.702250:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766485.702253:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3107 00000100:00080000:1.0:1775766485.702256:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766485.702261:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766485.702261:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967227136:192.168.202.146@tcp:38:kworker.0 00000008:00100000:1.0:1775766485.702269:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766485.702280:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967227392/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766485.702290:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766485.702302:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766485/real 1775766485] req@ffff8f994f2e6a00 x1862022967227136/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766485.702313:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967227136/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766485.702319:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766485.702323:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766485.702327:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967227136:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766485.702349:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967227264:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766485.702373:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766485.702386:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766485/real 1775766485] req@ffff8f994f2e5f80 x1862022967227264/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766485.702400:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967227264/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766485.702409:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766485.702412:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766485.702414:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967227264:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766485.702427:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967227392:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766485.702443:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766485.702451:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766485/real 1775766485] req@ffff8f994f2e6680 x1862022967227392/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766485.702475:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967227392/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766485.702481:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766485.702483:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766485.702486:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967227392:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766486.597804:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766486.597874:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766487.621891:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766487.621969:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766488.645840:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766488.645973:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766489.669900:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766489.669996:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766490.693876:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766490.694011:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766490.821879:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766490.821887:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766490.821891:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766490.821900:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3112 00000100:00080000:0.0:1775766490.821909:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766490.821915:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766490.821998:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967227520/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766490.822026:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766490.822030:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766490.822032:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766490.822038:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3112 00000100:00080000:0.0:1775766490.822042:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775766490.822058:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967227520:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775766490.822066:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766490.822076:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766490.822102:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967227648/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766490.822110:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766490.822113:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766490.822115:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766490.822118:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3112 00000100:00080000:0.0:1775766490.822121:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766490.822123:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766490.822127:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775766490.822129:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766490.822138:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967227776/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766490.822146:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766490.822155:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766490.822157:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766490.822160:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3112 00000100:00080000:0.0:1775766490.822162:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766490.822165:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766490.822168:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766490.822174:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766490/real 1775766490] req@ffff8f9970a74700 x1862022967227520/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766490.822187:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967227904/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766490.822209:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967227520/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766490.822221:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766490.822224:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766490.822229:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967227520:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766490.822250:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967227648:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766490.822271:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766490.822299:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766490/real 1775766490] req@ffff8f9970a76d80 x1862022967227648/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766490.822328:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967227648/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766490.822364:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766490.822367:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766490.822370:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967227648:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766490.822374:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967227776:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766490.822392:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766490.822400:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766490/real 1775766490] req@ffff8f9970a74380 x1862022967227776/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766490.822408:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967227776/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766490.822413:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766490.822415:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766490.822417:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967227776:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766490.822420:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967227904:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766490.822430:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766490.822435:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766490/real 1775766490] req@ffff8f9970a75c00 x1862022967227904/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766490.822442:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967227904/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766490.822447:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766490.822449:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766490.822463:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967227904:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766491.717839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766491.717900:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766492.741800:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766492.741908:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766493.765854:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766493.765907:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766494.789803:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766494.789893:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766495.813873:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766495.813934:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766495.941810:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766495.941819:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766495.941824:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766495.941832:0:38:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3117 00000100:00080000:0.0:1775766495.941841:0:38:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766495.941846:0:38:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766495.941877:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967228032/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766495.941902:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766495.941905:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766495.941906:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766495.941909:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3117 00000100:00080000:0.0:1775766495.941912:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766495.941914:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766495.941921:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766495.941929:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967228160/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766495.941936:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766495.941939:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766495.941940:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766495.941943:0:38:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3117 00000100:00080000:0.0:1775766495.941945:0:38:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766495.941948:0:38:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766495.941951:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766495.941958:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967228032:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766495.941959:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967228288/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766495.941971:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766495.941974:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766495.941975:0:38:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766495.941977:0:38:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3117 00000100:00080000:0.0:1775766495.941979:0:38:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766495.941980:0:38:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766495.941983:0:38:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766495.942002:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967228416/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766495.942036:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766495.942088:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766495/real 1775766495] req@ffff8f9970a77480 x1862022967228032/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766495.942109:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967228032/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766495.942117:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766495.942120:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766495.942123:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967228032:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766495.942127:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967228160:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766495.942142:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766495.942150:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766495/real 1775766495] req@ffff8f9970a77800 x1862022967228160/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766495.942157:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967228160/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766495.942163:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766495.942164:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766495.942166:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967228160:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766495.942190:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967228288:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766495.942203:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766495.942256:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766495/real 1775766495] req@ffff8f9970a76a00 x1862022967228288/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766495.942265:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967228288/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766495.942271:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766495.942273:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766495.942275:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967228288:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766495.942279:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967228416:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766495.942293:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766495.942299:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766495/real 1775766495] req@ffff8f9970a75500 x1862022967228416/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766495.942306:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967228416/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766495.942311:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766495.942313:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766495.942314:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967228416:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766496.837869:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766496.837930:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766497.861835:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766497.861886:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766498.885950:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766498.886081:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766499.909820:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766499.909914:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766500.933818:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766500.933952:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766501.061902:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766501.061914:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766501.061920:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766501.061941:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3122 00000100:00080000:2.0:1775766501.061952:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766501.061958:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766501.062321:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1500 x1862022967228544/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766501.062367:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766501.062372:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766501.062375:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766501.062395:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3122 00000100:00080000:2.0:1775766501.062407:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775766501.062409:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967228544:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775766501.062416:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766501.062430:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766501.062457:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0700 x1862022967228672/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766501.062469:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766501.062472:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766501.062476:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1775766501.062478:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775766501.062484:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3122 00000100:00080000:2.0:1775766501.062492:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766501.062497:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766501.062502:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766501.062525:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0e00 x1862022967228800/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766501.062535:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766501.062538:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766501.062540:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775766501.062545:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766501/real 1775766501] req@ffff8f99703d1500 x1862022967228544/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766501.062564:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3122 00000100:00080000:2.0:1775766501.062591:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766501.062598:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766501.062604:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766501.062612:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1500 x1862022967228544/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766501.062627:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766501.062632:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766501.062641:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2d80 x1862022967228928/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766501.062642:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967228544:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766501.062700:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967228672:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766501.062726:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766501.062761:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766501/real 1775766501] req@ffff8f99703d0700 x1862022967228672/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766501.062772:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0700 x1862022967228672/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766501.062795:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766501.062799:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766501.062801:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967228672:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766501.062808:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967228800:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766501.062829:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766501.062843:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766501/real 1775766501] req@ffff8f99703d0e00 x1862022967228800/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766501.062859:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0e00 x1862022967228800/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766501.062885:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766501.062890:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766501.062894:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967228800:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766501.062911:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967228928:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766501.062931:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766501.062940:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766501/real 1775766501] req@ffff8f99703d2d80 x1862022967228928/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766501.062949:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2d80 x1862022967228928/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766501.062956:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766501.062958:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766501.062961:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967228928:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766501.957842:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766501.957918:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766502.981871:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766502.981962:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766504.005847:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766504.005888:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766505.029895:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766505.029971:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766506.053850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766506.053871:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766506.181833:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766506.181843:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766506.181853:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766506.181865:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3127 00000100:00080000:2.0:1775766506.181876:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766506.181883:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766506.181915:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0380 x1862022967229056/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766506.181936:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766506.181939:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766506.181940:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766506.181943:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3127 00000100:00080000:2.0:1775766506.181946:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766506.181948:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766506.181956:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766506.181968:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0000 x1862022967229184/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766506.181969:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967229056:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775766506.181982:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766506.182033:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766506.182034:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766506.182037:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3127 00000100:00080000:2.0:1775766506.182055:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766506.182059:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766506.182065:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766506.182066:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766506.182097:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2680 x1862022967229312/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766506.182107:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766506.182112:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1775766506.182112:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766506/real 1775766506] req@ffff8f99703d0380 x1862022967229056/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766506.182138:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766506.182146:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3127 00000100:00080000:2.0:1775766506.182155:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766506.182161:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766506.182167:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766506.182169:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0380 x1862022967229056/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766506.182184:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3800 x1862022967229440/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766506.182184:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766506.182192:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766506.182199:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967229056:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766506.182221:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967229184:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766506.182240:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766506.182249:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766506/real 1775766506] req@ffff8f99703d0000 x1862022967229184/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766506.182258:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0000 x1862022967229184/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766506.182263:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766506.182265:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766506.182267:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967229184:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766506.182271:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967229312:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766506.182282:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766506.182292:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766506/real 1775766506] req@ffff8f99703d2680 x1862022967229312/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766506.182304:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2680 x1862022967229312/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766506.182319:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766506.182322:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766506.182326:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967229312:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766506.182332:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967229440:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766506.182351:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766506.182359:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766506/real 1775766506] req@ffff8f99703d3800 x1862022967229440/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766506.182367:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3800 x1862022967229440/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766506.182372:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766506.182374:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766506.182376:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967229440:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766507.077846:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766507.077946:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766508.101867:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766508.101885:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766509.125886:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766509.125964:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766510.149846:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766510.149902:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766511.173866:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766511.173959:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766511.301849:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766511.301860:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766511.301866:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766511.301877:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3132 00000100:00080000:3.0:1775766511.301890:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766511.301936:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766511.301978:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967229568/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766511.302002:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766511.302006:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766511.302007:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766511.302012:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3132 00000100:00080000:3.0:1775766511.302016:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766511.302018:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766511.302027:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766511.302037:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967229696/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766511.302046:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766511.302048:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766511.302050:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775766511.302055:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967229568:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775766511.302056:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3132 00000100:00080000:3.0:1775766511.302062:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766511.302078:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766511.302082:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766511.302094:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967229824/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766511.302102:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766511.302105:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766511.302106:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766511.302115:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3132 00000100:00080000:3.0:1775766511.302119:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766511.302123:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766511.302127:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766511.302134:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967229952/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766511.302160:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967229696:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766511.302208:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967229824:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766511.302246:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967229952:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766512.197824:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766512.197883:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766513.221872:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766513.221896:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766514.245855:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766514.245902:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1775766514.374002:0:2367:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:0.0:1775766514.374054:0:2367:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1775766514.374086:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775766514.374170:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775766514.374202:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775766514.374223:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:2.0:1775766514.374271:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766511/real 1775766514] req@ffff8f9970edf800 x1862022967229952/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766514.374330:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967229952/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766514.374346:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766514.374354:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766514.374360:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967229952:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766514.374376:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766511/real 1775766514] req@ffff8f9970edf480 x1862022967229696/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766514.374393:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967229696/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766514.374401:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766514.374405:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766514.374408:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967229696:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766514.374415:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766511/real 1775766514] req@ffff8f9970edd880 x1862022967229824/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766514.374427:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967229824/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766514.374435:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766514.374439:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766514.374458:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967229824:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766514.374466:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766511/real 1775766514] req@ffff8f9970ede300 x1862022967229568/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766514.374481:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967229568/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766514.374489:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766514.374492:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766514.374495:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967229568:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766515.269837:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766515.269893:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766516.293876:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766516.293886:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766516.421822:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766516.421835:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766516.421839:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766516.421850:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3138 00000100:00080000:3.0:1775766516.421862:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766516.421870:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766516.421913:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967230080/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766516.421939:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766516.421943:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766516.421945:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766516.421950:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3138 00000100:00080000:3.0:1775766516.421954:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766516.421959:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766516.421970:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766516.421981:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967230080:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766516.422038:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967230208/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766516.422049:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766516.422052:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766516.422053:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766516.422057:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3138 00000100:00080000:3.0:1775766516.422062:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766516.422066:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766516.422072:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766516.422085:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967230336/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766516.422095:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1775766516.422097:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775766516.422101:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766516.422104:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766516.422109:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3138 00000100:00080000:3.0:1775766516.422114:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766516.422119:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766516.422124:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766516.422136:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967230464/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766516.422165:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766516/real 1775766516] req@ffff8f9970edfb80 x1862022967230080/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766516.422186:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967230080/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766516.422201:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766516.422208:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766516.422213:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967230080:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766516.422238:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967230208:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766516.422264:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766516.422277:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766516/real 1775766516] req@ffff8f9970edc380 x1862022967230208/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766516.422316:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967230208/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766516.422325:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766516.422329:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766516.422332:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967230208:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766516.422340:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967230336:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766516.422363:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766516.422376:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766516/real 1775766516] req@ffff8f9970edca80 x1862022967230336/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766516.422388:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967230336/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766516.422395:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766516.422398:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766516.422401:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967230336:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766516.422407:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967230464:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766516.422425:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766516.422435:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766516/real 1775766516] req@ffff8f9970edea00 x1862022967230464/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766516.422445:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967230464/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766516.422474:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766516.422478:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766516.422481:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967230464:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766517.317852:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766517.317959:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766518.341860:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766518.341866:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766519.365855:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766519.365877:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766520.389884:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766520.389895:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766521.413819:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766521.413932:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766521.541911:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766521.541922:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766521.541927:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766521.541938:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3143 00000100:00080000:1.0:1775766521.541951:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766521.541981:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766521.542211:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967230592/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766521.542244:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766521.542256:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766521.542259:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766521.542263:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3143 00000100:00080000:1.0:1775766521.542273:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766521.542279:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766521.542288:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766521.542309:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967230592:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766521.542310:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967230720/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766521.542336:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766521.542338:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766521.542339:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766521.542341:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3143 00000100:00080000:1.0:1775766521.542344:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766521.542347:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766521.542350:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766521.542359:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967230848/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766521.542367:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766521.542370:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766521.542371:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766521.542374:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3143 00000100:00080000:1.0:1775766521.542380:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766521.542384:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766521.542402:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775766521.542409:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766521.542430:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967230976/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766521.542467:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766521/real 1775766521] req@ffff8f994f2e4700 x1862022967230592/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766521.542488:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967230592/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766521.542497:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766521.542501:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766521.542504:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967230592:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766521.542528:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967230720:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766521.542545:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766521.542553:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766521/real 1775766521] req@ffff8f994f2e6300 x1862022967230720/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766521.542561:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967230720/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766521.542567:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766521.542570:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766521.542614:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967230720:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766521.542619:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967230848:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766521.542634:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766521.542641:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766521/real 1775766521] req@ffff8f994f2e4e00 x1862022967230848/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766521.542650:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967230848/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766521.542655:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766521.542658:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766521.542660:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967230848:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766521.542665:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967230976:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766521.542675:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766521.542681:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766521/real 1775766521] req@ffff8f994f2e7b80 x1862022967230976/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766521.542689:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967230976/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766521.542694:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766521.542696:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766521.542697:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967230976:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766522.437936:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766522.438013:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766523.461879:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766523.461888:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766524.485854:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766524.485921:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766525.509866:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766525.509958:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766526.533848:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766526.533929:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766526.661818:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766526.661828:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766526.661832:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766526.661840:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3148 00000100:00080000:1.0:1775766526.661849:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766526.661854:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766526.661888:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967231104/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766526.661936:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766526.661939:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766526.661940:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766526.661944:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3148 00000100:00080000:1.0:1775766526.661946:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766526.661949:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766526.661958:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766526.661968:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967231232/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766526.661977:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775766526.661983:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967231104:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775766526.662015:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766526.662016:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766526.662020:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3148 00000100:00080000:1.0:1775766526.662022:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766526.662025:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766526.662028:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766526.662035:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967231360/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766526.662041:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766526.662043:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766526.662044:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766526.662047:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3148 00000100:00080000:1.0:1775766526.662049:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766526.662050:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766526.662053:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766526.662058:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967231488/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766526.662171:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766526.662241:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766526/real 1775766526] req@ffff8f994f2e7b80 x1862022967231104/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766526.662266:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967231104/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766526.662277:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766526.662281:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766526.662285:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967231104:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766526.662311:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967231232:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766526.662339:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766526.662349:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766526/real 1775766526] req@ffff8f994f2e4e00 x1862022967231232/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766526.662358:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967231232/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766526.662364:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766526.662366:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766526.662368:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967231232:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766526.662373:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967231360:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766526.662385:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766526.662391:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766526/real 1775766526] req@ffff8f994f2e6300 x1862022967231360/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766526.662423:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967231360/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766526.662430:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766526.662432:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766526.662434:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967231360:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766526.662439:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967231488:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766526.662451:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766526.662458:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766526/real 1775766526] req@ffff8f994f2e4700 x1862022967231488/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766526.662465:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967231488/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766526.662470:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766526.662472:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766526.662474:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967231488:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766527.557808:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766527.557865:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766528.581834:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766528.581935:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766529.605827:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766529.606000:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766530.629845:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766530.629918:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766531.653839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766531.653914:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766531.781888:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766531.781896:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766531.781901:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766531.781909:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3153 00000100:00080000:3.0:1775766531.781919:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766531.781925:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766531.781957:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd180 x1862022967231616/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766531.781984:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766531.781988:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766531.781989:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766531.781996:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3153 00000100:00080000:3.0:1775766531.781999:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766531.782003:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766531.782013:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766531.782026:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddf80 x1862022967231744/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766531.782036:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766531.782039:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766531.782041:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766531.782044:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3153 00000100:00100000:2.0:1775766531.782045:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967231616:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775766531.782048:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766531.782051:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766531.782056:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766531.782068:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967231872/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766531.782077:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766531.782080:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766531.782081:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766531.782084:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3153 00000100:00080000:3.0:1775766531.782095:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766531.782098:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766531.782102:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775766531.782104:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766531.782112:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967232000/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766531.782151:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766531/real 1775766531] req@ffff8f9970edd180 x1862022967231616/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766531.782173:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd180 x1862022967231616/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766531.782312:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766531.782319:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766531.782324:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967231616:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766531.782336:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967231744:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766531.782393:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766531.782417:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766531/real 1775766531] req@ffff8f9970eddf80 x1862022967231744/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766531.782428:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddf80 x1862022967231744/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766531.782435:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766531.782437:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766531.782439:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967231744:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766531.782464:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967231872:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766531.782477:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766531.782483:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766531/real 1775766531] req@ffff8f9970edea00 x1862022967231872/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766531.782491:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967231872/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766531.782496:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766531.782498:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766531.782500:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967231872:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766531.782524:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967232000:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766531.782537:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766531.782544:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766531/real 1775766531] req@ffff8f9970edca80 x1862022967232000/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766531.782552:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967232000/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766531.782557:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766531.782560:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766531.782562:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967232000:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766532.677855:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766532.677929:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766533.701894:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766533.701897:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766534.725858:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766534.725955:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766535.749894:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766535.749935:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766536.773838:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766536.773888:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766536.902492:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766536.902530:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766536.902534:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766536.902544:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3158 00000100:00080000:1.0:1775766536.902555:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766536.902566:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766536.902602:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967232128/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766536.902625:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766536.902628:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766536.902630:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766536.902632:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3158 00000100:00080000:1.0:1775766536.902635:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766536.902637:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766536.902644:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766536.902653:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967232256/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766536.902660:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766536.902663:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766536.902665:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766536.902668:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3158 00000100:00080000:1.0:1775766536.902670:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766536.902672:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766536.902676:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766536.902680:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967232128:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766536.902686:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967232384/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766536.902714:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766536.902718:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766536.902720:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766536.902724:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3158 00000100:00080000:1.0:1775766536.902729:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766536.902732:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766536.902777:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775766536.902782:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766536.902818:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967232512/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766536.902844:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766536/real 1775766536] req@ffff8f994f2e6680 x1862022967232128/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766536.902864:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967232128/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766536.902872:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766536.902875:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766536.902878:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967232128:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766536.902884:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967232256:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766536.902946:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766536.902973:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766536/real 1775766536] req@ffff8f994f2e5f80 x1862022967232256/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766536.902987:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967232256/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766536.902995:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766536.902998:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766536.903001:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967232256:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766536.903026:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967232384:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766536.903048:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766536.903060:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766536/real 1775766536] req@ffff8f994f2e6a00 x1862022967232384/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766536.903071:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967232384/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766536.903078:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766536.903081:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766536.903084:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967232384:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766536.903090:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967232512:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766536.903109:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766536.903119:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766536/real 1775766536] req@ffff8f994f2e4000 x1862022967232512/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766536.903154:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967232512/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766536.903162:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766536.903165:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766536.903169:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967232512:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775766537.797918:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766537.797996:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:0.0:1775766538.821859:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766538.821950:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766539.845908:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766539.846007:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766540.869834:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766540.869891:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766541.893912:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766541.894029:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766542.021847:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766542.021858:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766542.021863:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766542.021873:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3163 00000100:00080000:1.0:1775766542.021886:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766542.021893:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766542.021933:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967232640/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766542.021966:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766542.021969:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766542.021971:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766542.021997:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3163 00000100:00080000:1.0:1775766542.022002:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766542.022008:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766542.022009:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967232640:192.168.202.146@tcp:250:kworker.0 00000008:00100000:1.0:1775766542.022017:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766542.022032:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967232768/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766542.022042:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766542.022046:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766542.022047:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766542.022051:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3163 00000100:00080000:1.0:1775766542.022053:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766542.022057:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766542.022061:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766542.022070:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967232896/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766542.022078:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1775766542.022083:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775766542.022084:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766542.022087:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766542.022091:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3163 00000100:00080000:1.0:1775766542.022094:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766542.022097:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766542.022101:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766542.022112:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967233024/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766542.022128:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766542/real 1775766542] req@ffff8f994f2e5180 x1862022967232640/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766542.022183:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967232640/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766542.022194:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766542.022198:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766542.022202:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967232640:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766542.022227:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967232768:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766542.022251:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766542.022263:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766542/real 1775766542] req@ffff8f994f2e7800 x1862022967232768/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766542.022274:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967232768/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766542.022305:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766542.022308:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766542.022312:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967232768:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766542.022318:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967232896:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766542.022336:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766542.022347:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766542/real 1775766542] req@ffff8f994f2e4a80 x1862022967232896/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766542.022358:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967232896/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766542.022370:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766542.022374:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766542.022379:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967232896:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766542.022388:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967233024:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766542.022415:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766542.022454:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766542/real 1775766542] req@ffff8f994f2e6d80 x1862022967233024/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766542.022475:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967233024/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766542.022487:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766542.022506:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766542.022540:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967233024:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766542.917877:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766542.917916:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766543.941814:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766543.941864:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766544.965807:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766544.965860:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766545.989856:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766545.989916:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766547.013862:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766547.013919:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766547.141835:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766547.141844:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766547.141849:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766547.141859:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3168 00000100:00080000:1.0:1775766547.141879:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766547.141888:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766547.141928:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967233152/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766547.141960:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766547.141963:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766547.141965:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766547.141968:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3168 00000100:00080000:1.0:1775766547.141972:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766547.141975:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766547.141984:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766547.142001:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967233280/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766547.142011:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775766547.142015:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967233152:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775766547.142015:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766547.142018:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766547.142067:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3168 00000100:00080000:1.0:1775766547.142071:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766547.142074:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766547.142079:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766547.142090:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967233408/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766547.142097:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766547.142100:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766547.142102:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766547.142107:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3168 00000800:00000100:0.0:1775766547.142113:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775766547.142114:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766547.142120:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766547.142125:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766547.142137:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967233536/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766547.142175:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766547/real 1775766547] req@ffff8f994f2e6d80 x1862022967233152/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766547.142201:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967233152/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766547.142214:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766547.142219:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766547.142224:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967233152:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766547.142248:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967233280:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775766547.142270:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766547.142282:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766547/real 1775766547] req@ffff8f994f2e4a80 x1862022967233280/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766547.142293:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967233280/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766547.142320:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766547.142323:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766547.142326:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967233280:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775766547.142332:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967233408:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766547.142351:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766547.142363:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766547/real 1775766547] req@ffff8f994f2e7800 x1862022967233408/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766547.142374:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967233408/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766547.142383:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766547.142385:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766547.142387:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967233408:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766547.142393:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967233536:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766547.142411:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766547.142426:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766547/real 1775766547] req@ffff8f994f2e5180 x1862022967233536/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766547.142437:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967233536/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766547.142444:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766547.142447:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766547.142450:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967233536:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766548.037899:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766548.037935:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766549.061921:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766549.061974:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766550.085848:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766550.085888:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766551.109897:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766551.109952:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766552.133857:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766552.133877:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766552.261841:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766552.261850:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766552.261855:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766552.261863:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3173 00000100:00080000:3.0:1775766552.261873:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766552.261879:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766552.261927:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967233664/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766552.261966:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766552.261971:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766552.261974:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766552.261979:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3173 00000100:00080000:3.0:1775766552.261984:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766552.261988:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766552.261999:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766552.262015:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967233792/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766552.262029:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775766552.262040:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967233664:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775766552.262105:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766552.262109:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766552.262119:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3173 00000100:00080000:3.0:1775766552.262128:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775766552.262136:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775766552.262140:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766552.262169:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766552.262193:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967233920/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766552.262205:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766552/real 1775766552] req@ffff8f9970edc380 x1862022967233664/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766552.262221:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766552.262226:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766552.262229:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766552.262237:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3173 00000100:00100000:0.0:1775766552.262239:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967233664/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766552.262246:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766552.262257:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00080000:0.0:1775766552.262261:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1775766552.262264:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1775766552.262269:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766552.262276:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967233664:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766552.262301:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967234048/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766552.262320:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967233792:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775766552.262347:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766552.262361:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766552/real 1775766552] req@ffff8f9970edfb80 x1862022967233792/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766552.262371:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967233792/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766552.262377:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766552.262379:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766552.262383:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967233792:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775766552.262388:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967233920:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766552.262402:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766552.262409:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766552/real 1775766552] req@ffff8f9970ede300 x1862022967233920/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766552.262417:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967233920/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766552.262423:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766552.262425:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766552.262427:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967233920:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766552.262431:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967234048:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766552.262441:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766552.262447:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766552/real 1775766552] req@ffff8f9970edd880 x1862022967234048/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766552.262455:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967234048/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766552.262460:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766552.262462:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766552.262464:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967234048:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766553.157832:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766553.157920:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766554.181932:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766554.182006:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766555.205919:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766555.205969:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766556.229859:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766556.229936:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766557.253913:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766557.253936:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766557.381875:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766557.381885:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766557.381890:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766557.381901:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3178 00000100:00080000:2.0:1775766557.381915:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766557.381922:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766557.381969:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3800 x1862022967234176/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766557.381997:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766557.382000:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766557.382001:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766557.382005:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3178 00000100:00080000:2.0:1775766557.382007:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766557.382010:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766557.382020:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766557.382029:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2680 x1862022967234304/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766557.382078:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967234176:192.168.202.146@tcp:250:kworker.0 00000800:00000100:0.0:1775766557.382155:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775766557.382176:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766557.382180:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766557.382183:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766557.382189:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3178 00000100:00080000:2.0:1775766557.382194:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766557.382202:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766557.382209:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766557/real 1775766557] req@ffff8f99703d3800 x1862022967234176/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1775766557.382213:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766557.382249:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3800 x1862022967234176/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766557.382250:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0000 x1862022967234432/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766557.382268:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766557.382275:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766557.382280:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967234176:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775766557.382290:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766557.382295:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766557.382297:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766557.382302:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3178 00000100:00080000:2.0:1775766557.382308:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775766557.382312:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967234304:192.168.202.146@tcp:38:kworker.0 00000100:00080000:2.0:1775766557.382315:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766557.382324:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766557.382338:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0380 x1862022967234560/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766557.382347:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766557.382365:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766557/real 1775766557] req@ffff8f99703d2680 x1862022967234304/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766557.382378:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2680 x1862022967234304/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766557.382385:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766557.382389:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766557.382392:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967234304:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775766557.382430:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967234432:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766557.382451:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766557.382463:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766557/real 1775766557] req@ffff8f99703d0000 x1862022967234432/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766557.382476:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0000 x1862022967234432/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766557.382485:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766557.382488:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766557.382492:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967234432:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766557.382507:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967234560:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766557.382525:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766557.382534:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766557/real 1775766557] req@ffff8f99703d0380 x1862022967234560/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766557.382541:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0380 x1862022967234560/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766557.382547:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766557.382549:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766557.382551:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967234560:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766558.277840:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766558.277892:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766559.301832:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766559.301861:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766560.325872:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766560.325950:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766561.349802:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766561.349887:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766562.373821:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766562.373841:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766562.501857:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766562.501865:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766562.501870:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766562.501879:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3184 00000100:00080000:0.0:1775766562.501889:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766562.501894:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766562.501958:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967234688/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766562.501986:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766562.501990:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766562.501992:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766562.501997:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3184 00000100:00080000:0.0:1775766562.502002:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766562.502008:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766562.502018:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766562.502020:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967234688:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766562.502048:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967234816/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766562.502057:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766562.502060:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766562.502061:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766562.502064:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3184 00000100:00080000:0.0:1775766562.502068:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766562.502070:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766562.502076:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775766562.502081:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766562.502093:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967234944/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766562.502104:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766562.502108:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766562.502110:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766562.502115:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3184 00000100:00080000:0.0:1775766562.502119:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766562.502124:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766562.502130:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766562.502144:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766562/real 1775766562] req@ffff8f9970a77100 x1862022967234688/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766562.502146:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967235072/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766562.502173:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967234688/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766562.502182:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766562.502185:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766562.502190:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967234688:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766562.502210:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967234816:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766562.502226:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766562.502234:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766562/real 1775766562] req@ffff8f9970a76300 x1862022967234816/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766562.502242:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967234816/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766562.502264:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766562.502267:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766562.502270:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967234816:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766562.502286:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967234944:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766562.502302:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766562.502326:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766562/real 1775766562] req@ffff8f9970a75180 x1862022967234944/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766562.502334:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967234944/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766562.502364:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766562.502378:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766562.502381:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967234944:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766562.502385:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967235072:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766562.502400:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766562.502407:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766562/real 1775766562] req@ffff8f9970a75500 x1862022967235072/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766562.502415:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967235072/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766562.502421:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766562.502423:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766562.502441:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967235072:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766563.397866:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766563.397935:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766564.421900:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766564.421969:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766565.445931:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766565.445966:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766566.469864:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766566.469886:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766567.493950:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766567.493976:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766567.621840:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766567.621850:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766567.621856:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766567.621866:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3189 00000100:00080000:0.0:1775766567.621879:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766567.621886:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766567.621929:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967235200/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766567.621957:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766567.621961:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1775766567.622043:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967235200:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775766567.622063:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766567.622077:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3189 00000100:00080000:0.0:1775766567.622090:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766567.622099:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766567.622114:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775766567.622123:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766567.622154:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967235328/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766567.622188:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775766567.622188:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766567/real 1775766567] req@ffff8f9970a76a00 x1862022967235200/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766567.622204:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766567.622212:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766567.622221:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3189 00000100:00100000:2.0:1775766567.622227:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967235200/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766567.622231:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766567.622246:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766567.622253:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766567.622276:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967235456/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766567.622334:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766567.622338:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766567.622341:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00080000:0.0:1775766567.622349:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1775766567.622352:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967235200:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775766567.622354:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766567.622369:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3189 00000100:00080000:0.0:1775766567.622374:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766567.622377:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766567.622383:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766567.622391:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967235328:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766567.622414:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766567.622418:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967235584/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766567.622430:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766567/real 1775766567] req@ffff8f9970a77800 x1862022967235328/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766567.622441:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967235328/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766567.622452:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766567.622456:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766567.622458:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967235328:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766567.622464:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967235456:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766567.622493:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766567.622505:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766567/real 1775766567] req@ffff8f9970a77480 x1862022967235456/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766567.622517:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967235456/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766567.622525:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766567.622527:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766567.622530:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967235456:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766567.622556:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967235584:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766567.622571:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766567.622579:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766567/real 1775766567] req@ffff8f9970a75c00 x1862022967235584/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766567.622587:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967235584/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766567.622593:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766567.622595:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766567.622597:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967235584:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766568.517802:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766568.517913:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766569.541842:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766569.541904:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766570.565812:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766570.565832:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766571.589827:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766571.589927:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766572.613826:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766572.613917:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766572.741869:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766572.741881:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766572.741887:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766572.741900:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3194 00000100:00080000:3.0:1775766572.741915:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766572.741923:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766572.741965:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967235712/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766572.741999:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766572.742003:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766572.742006:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766572.742011:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3194 00000100:00080000:3.0:1775766572.742014:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766572.742018:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766572.742028:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766572.742055:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967235840/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766572.742065:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766572.742068:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766572.742070:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766572.742073:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3194 00000100:00080000:3.0:1775766572.742076:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766572.742079:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766572.742083:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766572.742094:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967235968/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766572.742113:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766572.742117:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766572.742119:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775766572.742126:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967235712:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775766572.742127:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3194 00000100:00080000:3.0:1775766572.742134:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766572.742140:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766572.742144:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766572.742158:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967236096/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766572.742207:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766572.742268:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766572/real 1775766572] req@ffff8f9970edf480 x1862022967235712/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766572.742296:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967235712/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766572.742307:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766572.742311:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766572.742317:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967235712:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766572.742325:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967235840:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766572.742698:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766572.742844:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766572/real 1775766572] req@ffff8f9970edf800 x1862022967235840/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766572.742874:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967235840/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766572.742889:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766572.742894:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766572.742899:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967235840:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766572.742913:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967235968:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766572.742957:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766572.742972:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766572/real 1775766572] req@ffff8f9970edc000 x1862022967235968/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766572.742987:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967235968/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766572.742997:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766572.743000:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766572.743004:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967235968:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766572.743058:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967236096:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766572.743088:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766572.743107:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766572/real 1775766572] req@ffff8f9970ede680 x1862022967236096/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766572.743191:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967236096/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766572.743202:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766572.743206:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766572.743210:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967236096:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766573.637940:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766573.637942:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766574.661818:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766574.661871:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766575.685833:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766575.685842:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766576.709885:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766576.710033:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766577.733857:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766577.733897:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766577.861888:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766577.861897:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766577.861902:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766577.861913:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3199 00000100:00080000:3.0:1775766577.861924:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766577.861931:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766577.861973:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967236224/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766577.862004:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766577.862008:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766577.862009:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766577.862014:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3199 00000100:00080000:3.0:1775766577.862018:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766577.862023:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766577.862051:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766577.862065:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967236352/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766577.862078:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775766577.862083:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967236224:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775766577.862083:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766577.862087:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766577.862100:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3199 00000100:00080000:3.0:1775766577.862120:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766577.862124:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766577.862130:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766577.862145:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967236480/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766577.862154:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766577.862158:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766577.862160:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766577.862163:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3199 00000100:00080000:3.0:1775766577.862167:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766577.862171:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766577.862176:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766577.862204:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967236608/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766577.862265:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967236352:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766577.862320:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967236480:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766577.862362:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967236608:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766578.757817:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766578.757918:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766579.781866:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766579.781926:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766580.805823:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766580.805839:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775766580.933976:0:2369:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:1.0:1775766580.934015:0:2369:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775766580.934032:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775766580.934119:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775766580.934130:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775766580.934137:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:2.0:1775766580.934449:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766577/real 1775766580] req@ffff8f9970edd500 x1862022967236352/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766580.934482:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967236352/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766580.934497:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766580.934504:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766580.934509:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967236352:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766580.934737:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766577/real 1775766580] req@ffff8f9970eded80 x1862022967236480/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766580.934858:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967236480/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766580.934879:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766580.934889:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766580.934898:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967236480:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766580.934915:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766577/real 1775766580] req@ffff8f9970edce00 x1862022967236608/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766580.934940:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967236608/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766580.934959:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766580.934967:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766580.934974:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967236608:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766580.934986:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766577/real 1775766580] req@ffff8f9970edc700 x1862022967236224/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766580.935006:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967236224/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766580.935019:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766580.935045:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766580.935052:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967236224:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766581.829823:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766581.830055:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766582.853833:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766582.853950:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766582.982075:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766582.982089:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766582.982097:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766582.982114:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3204 00000100:00080000:2.0:1775766582.982134:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766582.982145:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766582.982212:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2d80 x1862022967236736/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766582.982264:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766582.982270:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766582.982273:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766582.982280:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3204 00000100:00080000:2.0:1775766582.982288:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766582.982294:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766582.982354:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766582.982358:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967236736:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766582.982385:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0e00 x1862022967236864/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766582.982405:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766582.982414:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766582.982418:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766582.982427:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3204 00000100:00080000:2.0:1775766582.982436:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766582.982513:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766582.982527:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775766582.982539:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766582.982567:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0700 x1862022967236992/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766582.982591:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766582.982599:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766582.982604:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775766582.982611:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766582/real 1775766582] req@ffff8f99703d2d80 x1862022967236736/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766582.982622:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3204 00000100:00080000:2.0:1775766582.982641:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766582.982656:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766582.982669:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766582.982677:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2d80 x1862022967236736/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766582.982703:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1775766582.982705:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1500 x1862022967237120/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766582.982723:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766582.982813:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967236736:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766582.982857:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967236864:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766582.982898:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766582.982919:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766582/real 1775766582] req@ffff8f99703d0e00 x1862022967236864/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766582.982938:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0e00 x1862022967236864/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766582.982951:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766582.982956:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766582.982961:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967236864:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766582.982971:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967236992:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766582.983029:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766582.983047:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766582/real 1775766582] req@ffff8f99703d0700 x1862022967236992/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766582.983066:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0700 x1862022967236992/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766582.983078:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766582.983083:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766582.983088:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967236992:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766582.983098:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967237120:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766582.983133:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766582.983156:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766582/real 1775766582] req@ffff8f99703d1500 x1862022967237120/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766582.983172:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1500 x1862022967237120/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766582.983183:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766582.983188:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766582.983192:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967237120:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766583.877878:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766583.877958:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766584.901868:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766584.901981:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766585.925828:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766585.925851:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766586.949884:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766586.949985:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766587.973887:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766587.974003:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766588.101963:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766588.101973:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766588.101977:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766588.101985:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3209 00000100:00080000:0.0:1775766588.101995:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766588.102000:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766588.102065:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967237248/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766588.102099:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766588.102102:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766588.102103:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766588.102107:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3209 00000100:00080000:0.0:1775766588.102109:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766588.102112:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766588.102119:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766588.102127:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967237376/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766588.102133:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766588.102136:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766588.102137:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766588.102139:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3209 00000100:00080000:0.0:1775766588.102141:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766588.102143:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766588.102145:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766588.102150:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967237504/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766588.102156:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766588.102158:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766588.102159:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766588.102162:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3209 00000100:00080000:0.0:1775766588.102175:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766588.102178:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766588.102181:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766588.102185:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967237248:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766588.102193:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967237632/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766588.102252:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766588.102319:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766588/real 1775766588] req@ffff8f9970a74380 x1862022967237248/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766588.102365:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967237248/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766588.102376:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766588.102387:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766588.102390:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967237248:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766588.102395:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967237376:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766588.102413:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766588.102422:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766588/real 1775766588] req@ffff8f9970a76d80 x1862022967237376/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766588.102430:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967237376/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766588.102436:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766588.102438:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766588.102440:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967237376:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766588.102443:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967237504:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766588.102453:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766588.102458:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766588/real 1775766588] req@ffff8f9970a74700 x1862022967237504/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766588.102471:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967237504/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766588.102491:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766588.102493:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766588.102496:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967237504:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766588.102515:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967237632:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766588.102527:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766588.102534:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766588/real 1775766588] req@ffff8f9970a75c00 x1862022967237632/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766588.102541:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967237632/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766588.102546:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766588.102548:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766588.102550:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967237632:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766588.997877:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766588.997956:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766590.021851:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766590.021977:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766591.045878:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766591.045888:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766592.069832:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766592.069896:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766593.093827:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766593.093912:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766593.221878:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766593.221890:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766593.221895:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766593.221907:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3214 00000100:00080000:2.0:1775766593.221920:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766593.221927:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766593.222011:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3480 x1862022967237760/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766593.222037:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766593.222040:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766593.222042:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766593.222045:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3214 00000100:00080000:2.0:1775766593.222048:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766593.222051:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766593.222060:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766593.222069:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2300 x1862022967237888/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766593.222077:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766593.222081:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766593.222083:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775766593.222093:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967237760:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775766593.222094:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3214 00000100:00080000:2.0:1775766593.222107:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766593.222114:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766593.222121:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766593.222139:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1180 x1862022967238016/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766593.222151:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766593.222155:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766593.222158:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1775766593.222168:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775766593.222181:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3214 00000100:00080000:2.0:1775766593.222190:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766593.222196:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766593.222223:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766593.222234:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766593/real 1775766593] req@ffff8f99703d3480 x1862022967237760/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766593.222243:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3100 x1862022967238144/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766593.222263:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3480 x1862022967237760/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766593.222273:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766593.222277:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766593.222320:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967237760:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766593.222347:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967237888:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766593.222374:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766593.222388:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766593/real 1775766593] req@ffff8f99703d2300 x1862022967237888/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766593.222397:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2300 x1862022967237888/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766593.222403:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766593.222405:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766593.222408:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967237888:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766593.222412:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967238016:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766593.222426:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766593.222432:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766593/real 1775766593] req@ffff8f99703d1180 x1862022967238016/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766593.222439:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1180 x1862022967238016/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766593.222444:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766593.222446:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766593.222448:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967238016:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766593.222452:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967238144:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766593.222462:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766593.222483:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766593/real 1775766593] req@ffff8f99703d3100 x1862022967238144/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766593.222491:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3100 x1862022967238144/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766593.222496:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766593.222498:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766593.222501:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967238144:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766594.117905:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766594.118013:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766595.141859:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766595.141911:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766596.165857:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766596.165885:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:0.0:1775766597.189909:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766597.189990:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766598.213853:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766598.213996:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766598.341868:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766598.341877:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766598.341882:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766598.341891:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3219 00000100:00080000:1.0:1775766598.341901:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766598.341906:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766598.341937:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967238272/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766598.341965:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766598.341968:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766598.341969:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766598.341972:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3219 00000100:00080000:1.0:1775766598.341974:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766598.341976:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766598.341984:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766598.342022:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967238272:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766598.342034:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967238400/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766598.342042:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766598.342045:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766598.342046:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766598.342050:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3219 00000100:00080000:1.0:1775766598.342052:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766598.342055:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766598.342060:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766598.342068:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967238528/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766598.342075:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766598.342078:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766598.342079:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766598.342082:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3219 00000100:00080000:1.0:1775766598.342085:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766598.342088:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766598.342092:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766598.342099:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967238656/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766598.342137:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766598.342234:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766598/real 1775766598] req@ffff8f994f2e4000 x1862022967238272/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766598.342263:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967238272/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766598.342276:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766598.342281:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766598.342285:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967238272:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766598.342315:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967238400:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766598.342342:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766598.342355:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766598/real 1775766598] req@ffff8f994f2e6a00 x1862022967238400/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766598.342366:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967238400/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766598.342410:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766598.342413:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766598.342417:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967238400:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766598.342423:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967238528:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766598.342443:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766598.342452:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766598/real 1775766598] req@ffff8f994f2e5f80 x1862022967238528/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766598.342463:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967238528/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766598.342470:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766598.342473:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766598.342476:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967238528:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766598.342481:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967238656:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766598.342495:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766598.342503:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766598/real 1775766598] req@ffff8f994f2e6680 x1862022967238656/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766598.342513:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967238656/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766598.342519:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766598.342522:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766598.342550:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967238656:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766599.237824:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766599.237981:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766600.261858:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766600.262004:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766601.285842:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766601.285865:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766602.309827:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766602.309867:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766603.333818:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766603.333900:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766603.462412:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766603.462429:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766603.462434:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766603.462447:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3225 00000100:00080000:2.0:1775766603.462460:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766603.462469:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766603.462508:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1880 x1862022967238784/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766603.462545:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766603.462549:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766603.462551:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766603.462556:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3225 00000100:00080000:2.0:1775766603.462561:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766603.462565:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766603.462575:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766603.462591:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2a00 x1862022967238912/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766603.462602:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766603.462604:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766603.462606:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766603.462880:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3225 00000100:00080000:2.0:1775766603.462899:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766603.462912:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766603.462933:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766603.462988:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1c00 x1862022967239040/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766603.463017:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766603.463025:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766603.463029:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766603.463036:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3225 00000100:00080000:2.0:1775766603.463043:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766603.463052:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766603.463062:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766603.463085:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967238784:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766603.463090:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3b80 x1862022967239168/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775766603.463191:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766603.463260:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766603/real 1775766603] req@ffff8f99703d1880 x1862022967238784/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766603.463276:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1880 x1862022967238784/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766603.463285:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766603.463290:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766603.463294:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967238784:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766603.463303:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967238912:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766603.463327:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766603.463341:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766603/real 1775766603] req@ffff8f99703d2a00 x1862022967238912/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766603.463353:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2a00 x1862022967238912/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766603.463359:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766603.463362:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766603.463364:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967238912:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766603.463369:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967239040:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766603.463387:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766603.463394:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766603/real 1775766603] req@ffff8f99703d1c00 x1862022967239040/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766603.463402:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1c00 x1862022967239040/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766603.463427:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766603.463430:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766603.463432:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967239040:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766603.463453:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967239168:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766603.463467:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766603.463474:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766603/real 1775766603] req@ffff8f99703d3b80 x1862022967239168/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766603.463482:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3b80 x1862022967239168/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766603.463488:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766603.463490:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766603.463492:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967239168:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766604.357839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766604.357945:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775766605.381917:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766605.382010:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:0.0:1775766606.405903:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766606.405966:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766607.429832:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766607.429946:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766608.453838:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766608.453897:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766608.581846:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766608.581855:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766608.581859:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766608.581869:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3230 00000100:00080000:0.0:1775766608.581879:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766608.581884:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766608.581917:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967239296/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766608.581943:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766608.581946:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766608.581947:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766608.581951:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3230 00000100:00080000:0.0:1775766608.581953:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766608.581955:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766608.581962:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766608.581970:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967239424/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766608.581977:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766608.581980:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766608.581981:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766608.581984:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3230 00000100:00080000:0.0:1775766608.581985:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766608.581988:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766608.581991:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766608.581999:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967239552/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766608.582001:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967239296:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775766608.582007:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766608.582064:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766608.582065:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766608.582069:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3230 00000100:00080000:0.0:1775766608.582075:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766608.582078:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766608.582092:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775766608.582096:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766608.582105:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967239680/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766608.582136:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766608/real 1775766608] req@ffff8f9970a77480 x1862022967239296/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766608.582150:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967239296/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766608.582158:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766608.582161:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766608.582182:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967239296:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766608.582188:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967239424:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766608.582204:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766608.582212:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766608/real 1775766608] req@ffff8f9970a77800 x1862022967239424/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766608.582220:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967239424/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766608.582226:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766608.582228:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766608.582230:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967239424:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766608.582248:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967239552:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766608.582259:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766608.582265:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766608/real 1775766608] req@ffff8f9970a76a00 x1862022967239552/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766608.582272:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967239552/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766608.582277:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766608.582278:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766608.582280:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967239552:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766608.582284:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967239680:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766608.582293:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766608.582310:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766608/real 1775766608] req@ffff8f9970a75500 x1862022967239680/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766608.582318:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967239680/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766608.582323:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766608.582325:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766608.582327:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967239680:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766609.477817:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766609.477858:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766610.501825:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766610.501881:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766611.525831:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766611.525858:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766612.549838:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766612.549879:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766613.573830:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766613.573870:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766613.701883:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766613.701894:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766613.701898:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766613.701909:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3235 00000100:00080000:2.0:1775766613.701919:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766613.702113:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766613.702182:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3b80 x1862022967239808/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766613.702231:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766613.702237:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766613.702240:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766613.702248:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3235 00000100:00080000:2.0:1775766613.702260:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766613.702265:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766613.702281:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766613.702285:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967239808:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766613.702300:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1c00 x1862022967239936/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766613.702327:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766613.702332:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766613.702334:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766613.702340:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3235 00000100:00080000:2.0:1775766613.702344:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766613.702352:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:1.0:1775766613.702362:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:2.0:1775766613.702367:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766613.702389:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2a00 x1862022967240064/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766613.702402:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766613.702407:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1775766613.702413:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766613/real 1775766613] req@ffff8f99703d3b80 x1862022967239808/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766613.702430:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3b80 x1862022967239808/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766613.702438:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766613.702444:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766613.702446:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00080000:2.0:1775766613.702456:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3235 00000100:00080000:2.0:1775766613.702460:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775766613.702463:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967239808:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775766613.702469:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766613.702480:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766613.702495:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1880 x1862022967240192/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766613.702518:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967239936:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766613.702544:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766613.702565:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766613/real 1775766613] req@ffff8f99703d1c00 x1862022967239936/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766613.702616:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1c00 x1862022967239936/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766613.702623:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766613.702626:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766613.702644:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967239936:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766613.702651:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967240064:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766613.702673:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766613.702686:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766613/real 1775766613] req@ffff8f99703d2a00 x1862022967240064/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766613.702700:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2a00 x1862022967240064/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766613.702710:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766613.702715:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766613.702719:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967240064:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766613.702738:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967240192:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766613.702788:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766613.702809:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766613/real 1775766613] req@ffff8f99703d1880 x1862022967240192/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766613.702818:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1880 x1862022967240192/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766613.702845:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766613.702848:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766613.702850:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967240192:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766614.597830:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766614.597960:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766615.621878:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766615.621903:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766616.645884:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766616.645896:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766617.669883:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766617.669909:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775766618.693839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766618.693976:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766618.821845:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766618.821854:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766618.821859:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766618.821868:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3240 00000100:00080000:1.0:1775766618.821879:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766618.821884:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766618.821923:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967240320/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766618.821980:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766618.821983:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766618.821986:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766618.821992:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3240 00000100:00080000:1.0:1775766618.821998:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766618.822002:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766618.822011:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766618.822016:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967240320:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766618.822025:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967240448/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766618.822034:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766618.822052:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766618.822053:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766618.822056:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3240 00000100:00080000:1.0:1775766618.822060:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766618.822063:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766618.822067:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766618.822078:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967240576/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766618.822089:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775766618.822092:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766618.822096:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766618.822098:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766618.822103:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3240 00000100:00080000:1.0:1775766618.822109:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766618.822116:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766618.822122:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766618.822141:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967240704/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766618.822151:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766618/real 1775766618] req@ffff8f994f2e4700 x1862022967240320/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766618.822175:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967240320/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766618.822187:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766618.822196:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766618.822201:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967240320:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766618.822225:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967240448:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766618.822247:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766618.822264:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766618/real 1775766618] req@ffff8f994f2e6300 x1862022967240448/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766618.822373:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967240448/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766618.822385:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766618.822391:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766618.822398:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967240448:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766618.822411:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967240576:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766618.822484:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766618.822522:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766618/real 1775766618] req@ffff8f994f2e4e00 x1862022967240576/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766618.822542:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967240576/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766618.822549:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766618.822553:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766618.822556:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967240576:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766618.822562:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967240704:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766618.822577:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766618.822585:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766618/real 1775766618] req@ffff8f994f2e7b80 x1862022967240704/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766618.822592:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967240704/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766618.822621:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766618.822623:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766618.822626:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967240704:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766619.717923:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766619.717968:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766620.741886:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766620.741899:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766621.765866:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766621.765905:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775766622.789891:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766622.789947:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766623.813922:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766623.813936:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766623.941849:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766623.941857:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766623.941862:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766623.941870:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3245 00000100:00080000:1.0:1775766623.941881:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766623.941933:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766623.941969:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967240832/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766623.941991:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766623.941994:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766623.941995:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766623.941998:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3245 00000100:00080000:1.0:1775766623.942001:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766623.942004:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766623.942014:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766623.942030:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967240960/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766623.942040:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967240832:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775766623.942052:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766623.942056:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766623.942058:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766623.942064:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3245 00000100:00080000:1.0:1775766623.942069:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766623.942074:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766623.942080:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766623.942095:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967241088/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766623.942106:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766623.942109:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766623.942128:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766623.942133:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3245 00000100:00080000:1.0:1775766623.942139:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766623.942145:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:2.0:1775766623.942151:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:1.0:1775766623.942156:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766623.942174:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967241216/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766623.942222:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766623/real 1775766623] req@ffff8f994f2e7b80 x1862022967240832/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766623.942247:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967240832/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766623.942260:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766623.942266:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766623.942270:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967240832:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766623.942299:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967240960:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766623.942322:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766623.942333:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766623/real 1775766623] req@ffff8f994f2e4e00 x1862022967240960/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766623.942341:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967240960/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766623.942385:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766623.942388:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766623.942390:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967240960:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766623.942395:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967241088:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766623.942409:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766623.942417:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766623/real 1775766623] req@ffff8f994f2e6300 x1862022967241088/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766623.942424:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967241088/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766623.942430:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766623.942432:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766623.942434:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967241088:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766623.942438:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967241216:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766623.942448:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766623.942462:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766623/real 1775766623] req@ffff8f994f2e4700 x1862022967241216/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766623.942470:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967241216/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766623.942475:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766623.942477:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766623.942490:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967241216:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766624.837906:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766624.838015:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766625.861977:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766625.862041:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766626.885883:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766626.885896:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766627.909880:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766627.909957:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766628.933874:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766628.933955:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766629.061923:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766629.061933:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766629.061938:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766629.061951:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3250 00000100:00080000:1.0:1775766629.061964:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766629.061971:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766629.062016:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967241344/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766629.062045:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766629.062048:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766629.062050:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766629.062053:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3250 00000100:00080000:1.0:1775766629.062056:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766629.062059:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766629.062070:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766629.062101:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967241344:192.168.202.146@tcp:250:kworker.0 00000800:00000100:2.0:1775766629.062193:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766629.062239:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967241472/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766629.062260:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766629.062264:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766629.062270:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775766629.062270:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766629/real 1775766629] req@ffff8f994f2e6680 x1862022967241344/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766629.062285:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3250 00000100:00080000:1.0:1775766629.062303:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766629.062311:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766629.062320:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766629.062320:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967241344/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766629.062335:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967241600/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766629.062344:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766629.062346:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766629.062354:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766629.062357:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766629.062358:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00080000:1.0:1775766629.062367:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3250 00000100:00080000:1.0:1775766629.062371:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775766629.062376:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967241344:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775766629.062380:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766629.062388:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766629.062399:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967241728/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766629.062424:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967241472:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766629.062470:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766629.062487:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766629/real 1775766629] req@ffff8f994f2e5f80 x1862022967241472/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766629.062503:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967241472/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766629.062514:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766629.062517:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766629.062522:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967241472:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766629.062529:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967241600:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766629.062602:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766629.062616:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766629/real 1775766629] req@ffff8f994f2e6a00 x1862022967241600/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766629.062631:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967241600/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766629.062640:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766629.062645:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766629.062649:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967241600:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766629.062657:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967241728:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766629.062680:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766629.062692:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766629/real 1775766629] req@ffff8f994f2e4000 x1862022967241728/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766629.062706:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967241728/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766629.062724:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766629.062744:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766629.062748:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967241728:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766629.957859:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766629.957927:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766630.981828:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766630.981891:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766632.005883:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766632.005906:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766633.029914:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766633.029997:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766634.053872:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766634.053942:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766634.181858:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766634.181866:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766634.181908:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766634.181917:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3255 00000100:00080000:1.0:1775766634.181928:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766634.181934:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766634.181979:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967241856/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766634.182024:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766634.182029:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766634.182031:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766634.182035:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3255 00000100:00080000:1.0:1775766634.182038:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766634.182042:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766634.182050:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967241856:192.168.202.146@tcp:250:kworker.0 00000008:00100000:1.0:1775766634.182051:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766634.182067:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967241984/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766634.182075:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766634.182078:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766634.182079:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766634.182082:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3255 00000100:00080000:1.0:1775766634.182085:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766634.182087:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766634.182091:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766634.182098:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967242112/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766634.182115:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775766634.182125:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766634.182128:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766634.182130:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766634.182134:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3255 00000100:00080000:1.0:1775766634.182138:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766634.182140:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766634.182144:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766634.182153:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967242240/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766634.182164:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766634/real 1775766634] req@ffff8f994f2e5180 x1862022967241856/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766634.182301:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967241856/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766634.182313:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766634.182317:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766634.182321:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967241856:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766634.182356:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967241984:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766634.182412:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766634.182432:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766634/real 1775766634] req@ffff8f994f2e7800 x1862022967241984/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766634.182442:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967241984/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766634.182449:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766634.182452:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766634.182455:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967241984:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766634.182460:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967242112:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766634.182475:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766634.182482:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766634/real 1775766634] req@ffff8f994f2e4a80 x1862022967242112/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766634.182490:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967242112/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766634.182530:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766634.182533:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766634.182536:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967242112:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766634.182541:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967242240:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766634.182558:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766634.182566:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766634/real 1775766634] req@ffff8f994f2e6d80 x1862022967242240/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766634.182575:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967242240/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766634.182581:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766634.182583:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766634.182585:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967242240:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766635.077899:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766635.077996:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766636.101837:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766636.101907:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766637.125828:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766637.125833:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766638.149886:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766638.149913:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766639.173850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766639.173968:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766639.301855:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766639.301862:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766639.301866:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766639.301874:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3260 00000100:00080000:1.0:1775766639.301885:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766639.301891:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766639.301929:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967242368/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766639.301960:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766639.301963:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766639.301964:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766639.301967:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3260 00000100:00100000:2.0:1775766639.302005:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967242368:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775766639.302009:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766639.302021:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766639.302030:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766639.302046:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967242496/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766639.302058:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766639.302062:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766639.302064:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766639.302069:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3260 00000100:00080000:1.0:1775766639.302074:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766639.302077:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766639.302084:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766639.302098:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967242624/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766639.302103:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775766639.302118:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766639.302121:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766639.302122:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766639.302128:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3260 00000100:00080000:1.0:1775766639.302132:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766639.302137:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766639.302144:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766639.302157:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766639/real 1775766639] req@ffff8f994f2e6d80 x1862022967242368/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766639.302158:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967242752/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766639.302228:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967242368/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766639.302269:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766639.302277:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766639.302315:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967242368:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766639.302354:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967242496:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766639.302373:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766639.302382:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766639/real 1775766639] req@ffff8f994f2e4a80 x1862022967242496/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766639.302390:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967242496/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766639.302396:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766639.302399:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766639.302401:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967242496:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766639.302405:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967242624:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766639.302417:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766639.302423:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766639/real 1775766639] req@ffff8f994f2e7800 x1862022967242624/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766639.302430:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967242624/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766639.302435:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766639.302437:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766639.302439:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967242624:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766639.302442:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967242752:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766639.302453:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766639.302473:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766639/real 1775766639] req@ffff8f994f2e5180 x1862022967242752/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766639.302481:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967242752/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766639.302486:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766639.302489:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766639.302491:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967242752:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775766640.197903:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766640.198001:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:0.0:1775766641.221827:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766641.221885:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766642.245879:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766642.245902:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:0.0:1775766643.269876:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766643.269910:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766644.293834:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766644.293902:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766644.421892:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766644.421901:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766644.421905:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766644.421914:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3266 00000100:00080000:1.0:1775766644.421925:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766644.421931:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766644.421984:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967242880/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766644.422043:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766644.422048:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766644.422051:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766644.422056:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3266 00000100:00080000:1.0:1775766644.422063:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766644.422069:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766644.422080:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766644.422089:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967242880:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766644.422106:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967243008/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766644.422124:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766644.422136:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766644.422138:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766644.422144:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3266 00000100:00080000:1.0:1775766644.422218:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766644.422225:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766644.422242:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766644.422259:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967243136/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766644.422274:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766644.422278:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766644.422280:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766644.422286:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3266 00000100:00100000:2.0:1775766644.422291:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967243008:192.168.202.146@tcp:38:kworker.0 00000100:00080000:1.0:1775766644.422292:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766644.422303:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766644.422309:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766644.422325:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967243264/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766644.422339:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967243136:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766644.422378:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967243264:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766645.317871:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766645.317914:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766646.341836:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766646.341887:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766647.365864:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766647.365870:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000400:00000100:3.0:1775766647.494221:0:2366:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:3.0:1775766647.494272:0:2366:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775766647.494299:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:3.0:1775766647.494347:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:3.0:1775766647.494362:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:3.0:1775766647.494373:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:1.0:1775766647.494391:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766644/real 1775766647] req@ffff8f994f2e6a00 x1862022967243008/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766647.494421:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967243008/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766647.494433:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766647.494439:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766647.494444:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967243008:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766647.494476:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766644/real 1775766647] req@ffff8f994f2e5f80 x1862022967243136/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766647.494492:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967243136/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766647.494501:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766647.494505:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766647.494508:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967243136:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766647.494517:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766644/real 1775766647] req@ffff8f994f2e4000 x1862022967242880/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766647.494531:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967242880/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766647.494539:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766647.494543:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766647.494547:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967242880:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766647.494580:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766644/real 1775766647] req@ffff8f994f2e6680 x1862022967243264/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766647.494595:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967243264/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766647.494603:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766647.494606:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766647.494610:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967243264:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766648.389835:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766648.389856:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766649.413821:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766649.413887:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766649.541880:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766649.541889:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766649.541892:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766649.541901:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3271 00000100:00080000:0.0:1775766649.541910:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766649.541916:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766649.541955:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967243392/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766649.541976:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766649.541979:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766649.541981:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766649.541986:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3271 00000100:00080000:0.0:1775766649.541990:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766649.541994:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766649.542005:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766649.542021:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967243520/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766649.542022:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967243392:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775766649.542042:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766649.542046:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766649.542049:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766649.542053:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3271 00000100:00080000:0.0:1775766649.542057:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766649.542061:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766649.542073:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766649.542086:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967243648/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766649.542097:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1775766649.542101:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775766649.542103:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766649.542106:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766649.542112:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3271 00000100:00080000:0.0:1775766649.542117:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766649.542122:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766649.542128:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766649.542142:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967243776/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775766649.542153:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766649/real 1775766649] req@ffff8f9970a75180 x1862022967243392/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766649.542177:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967243392/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766649.542190:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766649.542194:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766649.542198:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967243392:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766649.542224:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967243520:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775766649.542249:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766649.542264:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766649/real 1775766649] req@ffff8f9970a76300 x1862022967243520/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766649.542277:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967243520/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766649.542287:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766649.542290:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766649.542294:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967243520:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775766649.542300:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967243648:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766649.542324:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766649.542335:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766649/real 1775766649] req@ffff8f9970a77100 x1862022967243648/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766649.542415:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967243648/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766649.542424:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766649.542427:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766649.542431:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967243648:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775766649.542437:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967243776:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775766649.542460:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766649.542472:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766649/real 1775766649] req@ffff8f9970a75500 x1862022967243776/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766649.542486:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967243776/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766649.542495:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766649.542499:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766649.542502:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967243776:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766650.437826:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766650.437921:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766651.461827:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766651.462073:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766652.485808:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766652.485862:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766653.509849:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766653.509882:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766654.533805:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766654.533847:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766654.661895:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766654.661904:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766654.661909:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766654.661932:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3276 00000100:00080000:1.0:1775766654.661944:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766654.661951:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766654.662033:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967243904/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766654.662062:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766654.662066:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766654.662069:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766654.662075:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3276 00000100:00080000:1.0:1775766654.662081:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766654.662088:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766654.662098:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967243904:192.168.202.146@tcp:250:kworker.0 00000008:00100000:1.0:1775766654.662099:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766654.662122:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967244032/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766654.662135:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766654.662139:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766654.662141:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766654.662146:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3276 00000100:00080000:1.0:1775766654.662169:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775766654.662174:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775766654.662178:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766654.662185:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766654.662200:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967244160/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766654.662210:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766654/real 1775766654] req@ffff8f994f2e6680 x1862022967243904/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766679 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766654.662213:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766654.662222:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766654.662225:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775766654.662234:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967243904/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766679 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766654.662242:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766654.662249:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3276 00000100:00080000:1.0:1775766654.662254:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766654.662258:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766654.662264:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766654.662276:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967244288/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766654.662365:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766654.662375:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967243904:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766654.662420:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967244032:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766654.662489:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766654.662515:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766654/real 1775766654] req@ffff8f994f2e4000 x1862022967244032/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766679 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766654.662531:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967244032/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766679 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766654.662541:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766654.662544:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766654.662547:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967244032:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766654.662552:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967244160:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766654.662607:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766654.662623:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766654/real 1775766654] req@ffff8f994f2e5f80 x1862022967244160/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766679 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766654.662641:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967244160/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766679 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766654.662652:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766654.662656:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766654.662659:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967244160:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766654.662666:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967244288:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766654.662720:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766654.662730:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766654/real 1775766654] req@ffff8f994f2e6a00 x1862022967244288/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766679 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766654.662795:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967244288/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766679 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766654.662822:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766654.662827:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766654.662838:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967244288:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766655.557827:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766655.557906:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766656.581876:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766656.582072:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766657.605808:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766657.605839:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766658.629856:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766658.629884:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766659.653802:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766659.653866:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766659.781896:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766659.781906:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766659.781911:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766659.781921:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3281 00000100:00080000:3.0:1775766659.781932:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766659.781938:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766659.781976:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddc00 x1862022967244416/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766659.782003:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766659.782007:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766659.782009:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766659.782013:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3281 00000100:00080000:3.0:1775766659.782016:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766659.782019:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766659.782028:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766659.782039:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf100 x1862022967244544/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766659.782049:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775766659.782053:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967244416:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775766659.782053:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766659.782057:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766659.782067:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3281 00000100:00080000:3.0:1775766659.782070:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766659.782072:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766659.782078:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766659.782091:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967244672/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766659.782099:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766659.782102:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766659.782105:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766659.782107:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3281 00000100:00080000:3.0:1775766659.782118:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766659.782121:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766659.782144:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775766659.782153:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766659.782158:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967244800/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766659.782222:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766659/real 1775766659] req@ffff8f9970eddc00 x1862022967244416/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766659.782252:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddc00 x1862022967244416/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766659.782266:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766659.782271:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766659.782275:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967244416:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766659.782317:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967244544:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766659.782346:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766659.782371:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766659/real 1775766659] req@ffff8f9970edf100 x1862022967244544/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766659.782507:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf100 x1862022967244544/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766659.782533:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766659.782536:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766659.782540:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967244544:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766659.782547:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967244672:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766659.782594:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766659.782610:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766659/real 1775766659] req@ffff8f9970edc700 x1862022967244672/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766659.782628:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967244672/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766659.782640:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766659.782645:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766659.782650:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967244672:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766659.782659:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967244800:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766659.782680:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766659.782690:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766659/real 1775766659] req@ffff8f9970edce00 x1862022967244800/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766659.782700:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967244800/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766659.782732:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766659.782735:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766659.782760:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967244800:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766660.677895:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766660.678004:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766661.701828:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766661.701938:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766662.725866:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766662.725901:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766663.749856:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766663.749895:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766664.773916:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766664.773992:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766664.901984:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766664.901993:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766664.901997:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766664.902007:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3286 00000100:00080000:1.0:1775766664.902018:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766664.902024:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766664.902057:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967244928/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766664.902084:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766664.902087:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766664.902088:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766664.902092:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3286 00000100:00080000:1.0:1775766664.902094:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766664.902097:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766664.902106:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766664.902151:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967244928:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766664.902155:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967245056/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766664.902179:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766664.902181:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766664.902182:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766664.902184:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3286 00000100:00080000:1.0:1775766664.902186:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766664.902190:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766664.902193:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766664.902206:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967245184/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766664.902212:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766664.902214:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766664.902215:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766664.902217:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3286 00000100:00080000:1.0:1775766664.902219:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766664.902221:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766664.902223:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766664.902229:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967245312/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766664.902498:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766664.902548:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766664/real 1775766664] req@ffff8f994f2e4700 x1862022967244928/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766664.902564:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967244928/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766664.902575:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766664.902579:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766664.902584:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967244928:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766664.902611:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967245056:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766664.902639:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766664.902648:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766664/real 1775766664] req@ffff8f994f2e6300 x1862022967245056/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766664.902656:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967245056/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766664.902661:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766664.902663:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766664.902665:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967245056:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766664.902669:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967245184:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766664.902683:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766664.902692:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766664/real 1775766664] req@ffff8f994f2e4e00 x1862022967245184/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766664.902704:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967245184/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766664.902719:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766664.902722:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766664.902726:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967245184:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766664.902733:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967245312:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766664.902784:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766664.902792:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766664/real 1775766664] req@ffff8f994f2e7b80 x1862022967245312/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766664.902800:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967245312/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766664.902806:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766664.902808:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766664.902810:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967245312:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766665.797900:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766665.797941:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766666.821809:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766666.821899:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766667.845820:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766667.845841:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766668.869827:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766668.869877:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766669.893866:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766669.894010:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766670.022146:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766670.022160:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766670.022167:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766670.022200:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3291 00000100:00080000:3.0:1775766670.022218:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766670.022229:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766670.022277:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967245440/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766670.022313:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766670.022319:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766670.022321:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766670.022327:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3291 00000100:00080000:3.0:1775766670.022334:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766670.022341:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766670.022353:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766670.022370:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967245440:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766670.022377:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967245568/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766670.022398:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766670.022403:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766670.022424:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766670.022430:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3291 00000100:00080000:3.0:1775766670.022436:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766670.022441:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766670.022449:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766670.022466:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967245696/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766670.022478:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766670.022483:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766670.022485:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766670.022493:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3291 00000800:00000100:2.0:1775766670.022494:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775766670.022500:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766670.022507:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766670.022514:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766670.022532:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967245824/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766670.022550:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766670/real 1775766670] req@ffff8f9970eded80 x1862022967245440/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766670.022579:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967245440/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766670.022594:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766670.022601:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766670.022608:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967245440:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766670.022660:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967245568:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766670.022696:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766670.022715:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766670/real 1775766670] req@ffff8f9970edd500 x1862022967245568/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766670.022732:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967245568/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766670.022779:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766670.022813:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766670.022817:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967245568:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766670.022823:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967245696:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766670.022848:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766670.022859:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766670/real 1775766670] req@ffff8f9970ede680 x1862022967245696/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766670.022869:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967245696/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766670.022876:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766670.022879:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766670.022882:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967245696:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766670.022887:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967245824:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766670.022901:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766670.022908:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766670/real 1775766670] req@ffff8f9970edc000 x1862022967245824/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766670.022917:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967245824/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766670.022924:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766670.022926:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766670.022928:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967245824:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766670.917833:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766670.917986:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766671.941839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766671.941988:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766672.965855:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766672.965917:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766673.989850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766673.989869:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775766675.013958:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766675.013966:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00080000:3.0:1775766675.141879:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766675.141892:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766675.141899:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766675.141912:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3296 00000100:00080000:3.0:1775766675.141926:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766675.141935:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766675.141980:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967245952/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766675.142020:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766675.142043:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766675.142046:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766675.142052:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3296 00000100:00080000:3.0:1775766675.142059:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766675.142066:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766675.142081:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766675.142109:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967245952:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766675.142120:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967246080/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766675.142138:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766675.142143:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766675.142147:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766675.142152:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3296 00000100:00080000:3.0:1775766675.142169:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766675.142175:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:2.0:1775766675.142182:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:3.0:1775766675.142188:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766675.142210:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967246208/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766675.142234:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766675.142240:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766675.142244:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775766675.142245:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766675/real 1775766675] req@ffff8f9970edf800 x1862022967245952/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766675.142256:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3296 00000100:00080000:3.0:1775766675.142268:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766675.142277:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766675.142283:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967245952/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1775766675.142289:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1775766675.142295:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766675.142300:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766675.142305:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967245952:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766675.142312:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967246336/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766675.142333:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967246080:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766675.142390:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766675.142425:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766675/real 1775766675] req@ffff8f9970edf480 x1862022967246080/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766675.142443:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967246080/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766675.142455:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766675.142460:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766675.142465:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967246080:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766675.142474:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967246208:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766675.142505:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766675.142520:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766675/real 1775766675] req@ffff8f9970edd880 x1862022967246208/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766675.142594:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967246208/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766675.142602:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766675.142605:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766675.142608:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967246208:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766675.142614:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967246336:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766675.142635:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766675.142644:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766675/real 1775766675] req@ffff8f9970ede300 x1862022967246336/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766675.142654:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967246336/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766675.142661:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766675.142664:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766675.142666:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967246336:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766676.037933:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766676.038000:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766677.061895:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766677.061912:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766678.085886:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766678.085977:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766679.109866:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766679.109947:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766680.133848:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766680.133916:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766680.261830:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766680.261838:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766680.261843:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766680.261851:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3301 00000100:00080000:0.0:1775766680.261861:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766680.261866:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766680.261899:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967246464/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766680.261923:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766680.261926:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766680.261927:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766680.261931:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3301 00000100:00080000:0.0:1775766680.261933:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766680.261935:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766680.261943:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766680.261952:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967246592/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766680.261960:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766680.261964:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1775766680.261964:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967246464:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775766680.261967:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766680.261975:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3301 00000100:00080000:0.0:1775766680.261987:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766680.261992:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766680.261999:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766680.262013:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967246720/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766680.262035:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775766680.262070:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775766680.262070:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766680/real 1775766680] req@ffff8f9970a76a00 x1862022967246464/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766680.262079:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766680.262084:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766680.262092:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3301 00000100:00100000:2.0:1775766680.262096:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967246464/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766680.262098:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766680.262105:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766680.262111:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1775766680.262123:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1775766680.262127:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967246848/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766680.262129:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766680.262135:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967246464:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766680.262158:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967246592:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766680.262176:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766680.262184:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766680/real 1775766680] req@ffff8f9970a77800 x1862022967246592/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766680.262193:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967246592/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766680.262198:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766680.262200:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766680.262202:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967246592:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766680.262206:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967246720:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766680.262216:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766680.262222:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766680/real 1775766680] req@ffff8f9970a77480 x1862022967246720/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766680.262229:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967246720/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766680.262234:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766680.262236:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766680.262237:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967246720:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766680.262253:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967246848:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766680.262265:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766680.262271:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766680/real 1775766680] req@ffff8f9970a75c00 x1862022967246848/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766680.262278:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967246848/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766680.262284:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766680.262286:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766680.262288:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967246848:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766681.157932:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766681.157968:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766682.181918:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766682.181945:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766683.205937:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766683.206034:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766684.229917:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766684.229964:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:3.0:1775766685.253890:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766685.253896:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766685.381874:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766685.381883:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766685.381888:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766685.381896:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3306 00000100:00080000:1.0:1775766685.381908:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766685.381913:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766685.381950:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967246976/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766685.381977:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766685.381980:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766685.381981:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766685.381984:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3306 00000100:00080000:1.0:1775766685.381987:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766685.381990:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766685.381998:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766685.382007:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967247104/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766685.382014:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766685.382017:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766685.382018:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766685.382021:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3306 00000100:00080000:1.0:1775766685.382023:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766685.382064:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766685.382064:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967246976:192.168.202.146@tcp:250:kworker.0 00000008:00100000:1.0:1775766685.382073:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766685.382084:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967247232/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766685.382092:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766685.382096:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766685.382097:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766685.382101:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3306 00000100:00080000:1.0:1775766685.382104:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766685.382107:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766685.382110:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766685.382118:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967247360/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766685.382132:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766685.382184:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766685/real 1775766685] req@ffff8f994f2e7b80 x1862022967246976/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766685.382200:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967246976/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766685.382209:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766685.382213:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766685.382216:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967246976:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766685.382222:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967247104:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766685.382295:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766685.382310:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766685/real 1775766685] req@ffff8f994f2e4e00 x1862022967247104/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766685.382322:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967247104/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766685.382330:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766685.382333:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766685.382336:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967247104:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766685.382361:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967247232:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766685.382378:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766685.382386:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766685/real 1775766685] req@ffff8f994f2e6300 x1862022967247232/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766685.382395:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967247232/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766685.382402:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766685.382405:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766685.382407:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967247232:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766685.382412:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967247360:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766685.382425:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766685.382432:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766685/real 1775766685] req@ffff8f994f2e4700 x1862022967247360/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766685.382466:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967247360/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766685.382474:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766685.382476:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766685.382496:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967247360:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766686.277854:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766686.277903:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766687.301887:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766687.301956:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766688.325861:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766688.325894:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766689.349854:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766689.349880:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766690.373908:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766690.373978:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766690.501850:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766690.501860:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766690.501865:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766690.501876:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3312 00000100:00080000:1.0:1775766690.501888:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766690.501896:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766690.501941:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967247488/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766690.501977:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766690.502015:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766690.502017:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766690.502023:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3312 00000100:00100000:3.0:1775766690.502024:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967247488:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775766690.502029:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766690.502034:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766690.502045:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766690.502061:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967247616/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766690.502070:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766690.502072:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766690.502074:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766690.502081:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3312 00000100:00080000:1.0:1775766690.502084:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766690.502086:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:3.0:1775766690.502090:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:1.0:1775766690.502091:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766690.502103:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967247744/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766690.502112:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766690.502115:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766690.502117:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766690.502119:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3312 00000100:00080000:1.0:1775766690.502123:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766690.502126:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766690.502142:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766690.502143:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766690/real 1775766690] req@ffff8f994f2e6a00 x1862022967247488/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766690.502155:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967247872/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766690.502164:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967247488/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766690.502249:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766690.502256:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766690.502261:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967247488:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766690.502285:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967247616:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766690.502348:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766690.502358:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766690/real 1775766690] req@ffff8f994f2e5f80 x1862022967247616/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766690.502367:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967247616/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766690.502492:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766690.502494:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766690.502497:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967247616:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766690.502502:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967247744:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766690.502589:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766690.502607:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766690/real 1775766690] req@ffff8f994f2e4000 x1862022967247744/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766690.502615:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967247744/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766690.502620:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766690.502622:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766690.502625:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967247744:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766690.502629:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967247872:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766690.502641:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766690.502647:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766690/real 1775766690] req@ffff8f994f2e6680 x1862022967247872/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766690.502654:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967247872/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766690.502659:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766690.502660:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766690.502662:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967247872:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766691.397840:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766691.397866:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766692.421853:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766692.421907:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766693.445831:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766693.445954:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766694.469893:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766694.469903:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766695.493870:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766695.493962:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766695.621866:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766695.621875:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766695.621879:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766695.621887:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3317 00000100:00080000:0.0:1775766695.621896:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766695.621902:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766695.621934:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967248000/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766695.621955:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766695.621958:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766695.621959:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766695.621962:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3317 00000100:00080000:0.0:1775766695.621965:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766695.621967:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766695.621975:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766695.621982:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967248128/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766695.621990:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766695.621992:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766695.621993:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766695.621996:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3317 00000100:00080000:0.0:1775766695.621998:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766695.622000:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766695.622003:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766695.622010:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967248000:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766695.622014:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967248256/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766695.622022:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766695.622025:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766695.622026:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766695.622029:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3317 00000100:00080000:0.0:1775766695.622031:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766695.622033:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:3.0:1775766695.622069:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:0.0:1775766695.622075:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766695.622086:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967248384/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766695.622115:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766695/real 1775766695] req@ffff8f9970a74700 x1862022967248000/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766695.622131:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967248000/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766695.622154:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766695.622158:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766695.622161:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967248000:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766695.622167:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967248128:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766695.622184:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766695.622193:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766695/real 1775766695] req@ffff8f9970a76d80 x1862022967248128/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766695.622201:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967248128/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766695.622207:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766695.622209:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766695.622211:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967248128:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766695.622251:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967248256:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766695.622267:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766695.622274:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766695/real 1775766695] req@ffff8f9970a74380 x1862022967248256/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766695.622284:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967248256/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766695.622291:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766695.622293:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766695.622296:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967248256:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766695.622300:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967248384:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766695.622325:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766695.622332:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766695/real 1775766695] req@ffff8f9970a75c00 x1862022967248384/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766695.622342:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967248384/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766695.622349:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766695.622352:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766695.622354:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967248384:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766696.517807:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766696.517879:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766697.541868:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766697.541964:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766698.565868:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766698.566006:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766699.589868:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766699.590056:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766700.613858:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766700.613909:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766700.741911:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766700.741924:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766700.741930:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766700.741942:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3322 00000100:00080000:0.0:1775766700.741957:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766700.741964:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766700.742014:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967248512/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766700.742039:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766700.742044:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766700.742046:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766700.742051:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3322 00000100:00080000:0.0:1775766700.742055:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766700.742060:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766700.742072:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766700.742100:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967248640/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766700.742109:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967248512:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775766700.742128:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766700.742131:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766700.742133:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766700.742136:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3322 00000100:00080000:0.0:1775766700.742140:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766700.742143:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766700.742148:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766700.742158:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967248768/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766700.742164:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766700.742167:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766700.742169:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766700.742171:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3322 00000100:00080000:0.0:1775766700.742173:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766700.742175:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766700.742178:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766700.742185:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967248896/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766700.742202:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766700.742260:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766700/real 1775766700] req@ffff8f9970a77480 x1862022967248512/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766700.742278:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967248512/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766700.742295:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766700.742299:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766700.742302:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967248512:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766700.742325:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967248640:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766700.742343:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766700.742351:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766700/real 1775766700] req@ffff8f9970a77800 x1862022967248640/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766700.742359:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967248640/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766700.742365:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766700.742368:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766700.742370:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967248640:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766700.742375:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967248768:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766700.742386:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766700.742392:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766700/real 1775766700] req@ffff8f9970a76a00 x1862022967248768/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766700.742399:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967248768/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766700.742405:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766700.742406:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766700.742408:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967248768:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766700.742411:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967248896:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766700.742422:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766700.742427:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766700/real 1775766700] req@ffff8f9970a75500 x1862022967248896/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766700.742434:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967248896/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766700.742439:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766700.742441:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766700.742443:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967248896:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766701.637855:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766701.637905:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766702.661878:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766702.661970:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766703.685879:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766703.685896:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766704.709860:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766704.709908:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766705.733851:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766705.733916:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766705.861841:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766705.861864:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766705.861869:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766705.861879:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3327 00000100:00080000:1.0:1775766705.861888:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766705.861894:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766705.861942:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967249024/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766705.861981:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766705.861985:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766705.861987:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766705.861991:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3327 00000100:00080000:1.0:1775766705.861995:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766705.861999:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766705.862024:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967249024:192.168.202.146@tcp:250:kworker.0 00000008:00100000:1.0:1775766705.862058:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766705.862073:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967249152/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766705.862092:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766705.862095:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766705.862096:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766705.862100:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3327 00000800:00000100:3.0:1775766705.862105:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775766705.862107:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766705.862111:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766705.862116:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766705.862125:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967249280/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766705.862131:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766705.862134:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766705.862135:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766705.862137:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3327 00000100:00080000:1.0:1775766705.862139:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766705.862142:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766705.862145:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766705.862149:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766705/real 1775766705] req@ffff8f994f2e5180 x1862022967249024/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766705.862157:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967249408/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766705.862171:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967249024/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766705.862179:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766705.862182:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766705.862186:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967249024:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766705.862205:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967249152:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766705.862220:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766705.862228:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766705/real 1775766705] req@ffff8f994f2e7800 x1862022967249152/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766705.862237:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967249152/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766705.862246:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766705.862250:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766705.862253:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967249152:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766705.862259:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967249280:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766705.862280:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766705.862291:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766705/real 1775766705] req@ffff8f994f2e4a80 x1862022967249280/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766705.862303:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967249280/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766705.862313:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766705.862315:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766705.862318:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967249280:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766705.862321:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967249408:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766705.862335:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766705.862341:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766705/real 1775766705] req@ffff8f994f2e6d80 x1862022967249408/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766705.862358:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967249408/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766705.862363:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766705.862364:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766705.862366:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967249408:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766706.757850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766706.757951:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766707.781824:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766707.781927:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766708.805807:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766708.805899:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766709.829826:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766709.829881:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766710.853816:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766710.853869:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766710.981876:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766710.981886:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766710.981890:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766710.981908:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3332 00000100:00080000:2.0:1775766710.981919:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766710.981925:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766710.981963:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3100 x1862022967249536/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766710.982035:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766710.982039:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766710.982079:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766710.982083:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3332 00000100:00080000:2.0:1775766710.982087:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775766710.982089:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967249536:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775766710.982093:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766710.982105:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766710.982118:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1180 x1862022967249664/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766710.982126:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766710.982129:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766710.982130:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766710.982133:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3332 00000100:00080000:2.0:1775766710.982142:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766710.982145:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766710.982149:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766710.982157:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2300 x1862022967249792/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766710.982164:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766710.982166:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766710.982168:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766710.982170:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3332 00000100:00080000:2.0:1775766710.982172:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766710.982174:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766710.982178:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766710.982204:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967249664:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766710.982210:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3480 x1862022967249920/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766710.982256:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967249792:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766710.982285:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967249920:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766711.877837:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766711.877942:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766712.901835:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766712.901910:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766713.925810:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766713.925850:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775766714.054024:0:2368:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:2.0:1775766714.054063:0:2368:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775766714.054081:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:2.0:1775766714.054129:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:2.0:1775766714.054140:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:2.0:1775766714.054151:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:3.0:1775766714.054180:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766710/real 1775766714] req@ffff8f99703d3480 x1862022967249920/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766714.054208:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3480 x1862022967249920/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766714.054221:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766714.054225:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766714.054230:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967249920:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766714.054238:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766710/real 1775766714] req@ffff8f99703d1180 x1862022967249664/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766714.054252:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1180 x1862022967249664/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766714.054261:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766714.054265:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766714.054269:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967249664:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766714.054277:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766710/real 1775766714] req@ffff8f99703d2300 x1862022967249792/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766714.054288:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2300 x1862022967249792/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766714.054295:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766714.054298:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766714.054302:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967249792:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766714.054310:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766710/real 1775766714] req@ffff8f99703d3100 x1862022967249536/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766714.054323:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3100 x1862022967249536/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766714.054333:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766714.054337:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766714.054348:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967249536:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766714.949821:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766714.949878:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766715.973828:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766715.973947:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766716.101828:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766716.101839:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766716.101843:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766716.101855:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3337 00000100:00080000:2.0:1775766716.101878:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766716.101886:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766716.101939:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1500 x1862022967250048/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766716.101975:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766716.101978:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766716.101980:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766716.101983:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3337 00000100:00080000:2.0:1775766716.101986:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766716.101989:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766716.101998:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766716.102053:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967250048:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766716.102068:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0700 x1862022967250176/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766716.102081:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766716.102085:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766716.102087:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766716.102093:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3337 00000100:00080000:2.0:1775766716.102098:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766716.102103:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766716.102109:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766716.102124:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0e00 x1862022967250304/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766716.102136:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775766716.102151:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766716.102155:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766716.102158:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766716.102163:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3337 00000100:00080000:2.0:1775766716.102177:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766716.102181:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766716.102186:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766716.102199:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2d80 x1862022967250432/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766716.102213:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766716/real 1775766716] req@ffff8f99703d1500 x1862022967250048/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766716.102238:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1500 x1862022967250048/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766716.102251:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766716.102257:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766716.102261:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967250048:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766716.102286:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967250176:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766716.102313:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766716.102327:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766716/real 1775766716] req@ffff8f99703d0700 x1862022967250176/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766716.102340:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0700 x1862022967250176/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766716.102350:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766716.102353:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766716.102357:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967250176:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766716.102364:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967250304:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766716.102387:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766716.102398:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766716/real 1775766716] req@ffff8f99703d0e00 x1862022967250304/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766716.102410:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0e00 x1862022967250304/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766716.102419:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766716.102423:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766716.102426:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967250304:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766716.102432:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967250432:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766716.102452:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766716.102463:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766716/real 1775766716] req@ffff8f99703d2d80 x1862022967250432/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766716.102475:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2d80 x1862022967250432/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766716.102483:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766716.102486:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766716.102490:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967250432:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766716.997834:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766716.997953:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766718.021975:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766718.022024:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766719.045832:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766719.045889:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766720.069819:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766720.069908:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766721.093825:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766721.093904:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766721.221834:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766721.221845:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766721.221849:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766721.221895:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3342 00000100:00080000:1.0:1775766721.221905:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766721.221911:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766721.221962:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967250560/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766721.221991:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766721.221994:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766721.221995:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766721.221998:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3342 00000100:00080000:1.0:1775766721.222002:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766721.222004:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766721.222013:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766721.222023:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967250688/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766721.222032:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766721.222034:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1775766721.222039:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967250560:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775766721.222072:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766721.222076:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3342 00000100:00080000:1.0:1775766721.222079:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766721.222081:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766721.222086:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766721.222093:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766721.222097:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967250816/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766721.222105:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766721.222107:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766721.222108:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766721.222111:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3342 00000100:00080000:1.0:1775766721.222114:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766721.222116:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766721.222119:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766721.222126:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967250944/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766721.222138:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766721/real 1775766721] req@ffff8f994f2e6d80 x1862022967250560/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766721.222156:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967250560/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766721.222164:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766721.222167:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766721.222169:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967250560:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766721.222188:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967250688:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766721.222201:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766721.222209:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766721/real 1775766721] req@ffff8f994f2e4a80 x1862022967250688/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766721.222217:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967250688/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766721.222222:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766721.222224:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766721.222226:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967250688:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766721.222229:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967250816:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766721.222239:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766721.222245:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766721/real 1775766721] req@ffff8f994f2e7800 x1862022967250816/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766721.222252:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967250816/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766721.222269:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766721.222272:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766721.222274:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967250816:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766721.222281:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967250944:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766721.222292:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766721.222298:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766721/real 1775766721] req@ffff8f994f2e5180 x1862022967250944/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766721.222306:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967250944/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766721.222310:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766721.222312:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766721.222314:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967250944:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766722.117858:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766722.117989:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766723.141915:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766723.141950:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766724.165827:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766724.165985:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766725.189847:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766725.189901:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766726.213899:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766726.213974:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766726.341900:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766726.341907:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766726.341912:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766726.341921:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3347 00000100:00080000:0.0:1775766726.341931:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766726.341936:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766726.341968:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967251072/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766726.341991:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766726.341996:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766726.341998:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766726.342002:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3347 00000100:00080000:0.0:1775766726.342005:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766726.342008:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766726.342017:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766726.342026:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967251200/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766726.342034:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967251072:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775766726.342039:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766726.342044:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766726.342045:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766726.342049:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3347 00000100:00080000:0.0:1775766726.342052:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766726.342055:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766726.342059:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766726.342067:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967251328/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766726.342073:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766726.342076:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766726.342077:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766726.342080:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3347 00000100:00080000:0.0:1775766726.342083:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766726.342085:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:2.0:1775766726.342105:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:0.0:1775766726.342126:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766726.342140:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967251456/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766726.342140:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766726/real 1775766726] req@ffff8f9970a77100 x1862022967251072/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766726.342181:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967251072/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766726.342190:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766726.342193:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766726.342197:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967251072:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766726.342214:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967251200:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766726.342231:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766726.342238:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766726/real 1775766726] req@ffff8f9970a76300 x1862022967251200/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766726.342247:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967251200/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766726.342253:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766726.342256:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766726.342259:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967251200:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766726.342265:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967251328:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766726.342285:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766726.342296:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766726/real 1775766726] req@ffff8f9970a75180 x1862022967251328/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766726.342338:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967251328/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766726.342347:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766726.342350:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766726.342356:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967251328:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766726.342364:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967251456:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766726.342387:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766726.342402:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766726/real 1775766726] req@ffff8f9970a75500 x1862022967251456/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766726.342415:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967251456/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766726.342423:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766726.342426:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766726.342429:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967251456:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766727.237876:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766727.237930:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766728.261926:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766728.261951:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766729.285839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766729.285909:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766730.309886:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766730.309954:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766731.333870:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766731.333959:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766731.461880:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766731.461891:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766731.461896:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766731.462010:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3353 00000100:00080000:1.0:1775766731.462022:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766731.462028:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766731.462075:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967251584/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766731.462105:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766731.462113:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766731.462115:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766731.462118:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3353 00000100:00080000:1.0:1775766731.462121:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766731.462123:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766731.462131:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766731.462141:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967251712/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766731.462149:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766731.462151:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766731.462153:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766731.462157:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3353 00000100:00100000:2.0:1775766731.462158:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967251584:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775766731.462159:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766731.462166:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766731.462170:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766731.462179:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967251840/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766731.462228:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766731.462231:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766731.462233:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766731.462237:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3353 00000100:00080000:1.0:1775766731.462242:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775766731.462247:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775766731.462249:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766731.462257:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766731.462270:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967251968/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766731.462294:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766731/real 1775766731] req@ffff8f994f2e6680 x1862022967251584/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766731.462311:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967251584/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766731.462337:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766731.462340:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766731.462343:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967251584:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766731.462362:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967251712:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766731.462381:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766731.462393:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766731/real 1775766731] req@ffff8f994f2e4000 x1862022967251712/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766731.462406:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967251712/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766731.462415:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766731.462418:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766731.462422:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967251712:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766731.462429:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967251840:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766731.462451:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766731.462462:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766731/real 1775766731] req@ffff8f994f2e5f80 x1862022967251840/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766731.462475:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967251840/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766731.462485:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766731.462488:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766731.462492:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967251840:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766731.462512:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967251968:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766731.462529:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766731.462537:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766731/real 1775766731] req@ffff8f994f2e6a00 x1862022967251968/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766731.462545:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967251968/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766731.462551:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766731.462553:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766731.462555:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967251968:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766732.357878:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766732.357941:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766733.381901:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766733.381965:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766734.405898:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766734.405960:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766735.429857:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766735.429911:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766736.453949:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766736.453989:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766736.582547:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766736.582584:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766736.582593:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766736.582613:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3358 00000100:00080000:0.0:1775766736.582650:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766736.582664:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766736.583018:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967252096/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766736.583064:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766736.583071:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766736.583075:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766736.583083:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3358 00000100:00080000:0.0:1775766736.583091:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766736.583102:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766736.583120:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766736.583145:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967252224/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766736.583147:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967252096:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775766736.583191:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766736.583195:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766736.583198:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766736.583204:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3358 00000100:00080000:0.0:1775766736.583210:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766736.583216:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766736.583224:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766736.583244:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967252352/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766736.583257:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766736.583262:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766736.583267:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766736.583274:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3358 00000100:00080000:0.0:1775766736.583282:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775766736.583291:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775766736.583299:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766736.583310:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766736.583338:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967252480/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766736.583360:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766736/real 1775766736] req@ffff8f9970a76a00 x1862022967252096/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766761 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766736.583388:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967252096/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766761 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766736.583404:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766736.583410:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766736.583416:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967252096:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766736.583450:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967252224:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766736.583483:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766736.583517:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766736/real 1775766736] req@ffff8f9970a77800 x1862022967252224/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766761 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766736.583545:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967252224/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766761 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766736.583576:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766736.583582:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766736.583588:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967252224:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766736.583599:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967252352:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766736.583637:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766736.583656:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766736/real 1775766736] req@ffff8f9970a77480 x1862022967252352/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766761 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766736.583670:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967252352/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766761 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766736.583681:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766736.583685:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766736.583688:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967252352:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766736.583695:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967252480:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766736.583786:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766736.583811:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766736/real 1775766736] req@ffff8f9970a75c00 x1862022967252480/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766761 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766736.583846:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967252480/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766761 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766736.583861:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766736.583866:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766736.583870:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967252480:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766737.477851:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766737.477937:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766738.501875:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766738.501972:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766739.525836:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766739.525902:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766740.549898:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766740.549972:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766741.573818:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766741.573871:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766741.701849:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766741.701857:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766741.701861:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766741.701870:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3363 00000100:00080000:0.0:1775766741.701882:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766741.701889:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766741.701929:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967252608/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766741.701959:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766741.701962:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766741.701963:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766741.701967:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3363 00000100:00080000:0.0:1775766741.701972:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766741.701977:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766741.701988:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766741.702005:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967252608:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766741.702007:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967252736/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766741.702028:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766741.702033:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766741.702035:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766741.702041:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3363 00000100:00080000:0.0:1775766741.702045:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766741.702049:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766741.702055:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766741.702066:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967252864/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766741.702073:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766741.702075:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766741.702077:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766741.702080:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3363 00000100:00080000:0.0:1775766741.702091:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766741.702094:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:2.0:1775766741.702102:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:0.0:1775766741.702105:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766741.702120:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967252992/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766741.702166:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766741/real 1775766741] req@ffff8f9970a74380 x1862022967252608/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766741.702184:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967252608/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766741.702195:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766741.702201:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766741.702205:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967252608:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766741.702231:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967252736:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766741.702249:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766741.702257:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766741/real 1775766741] req@ffff8f9970a76d80 x1862022967252736/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766741.702265:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967252736/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766741.702404:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766741.702409:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766741.702413:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967252736:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766741.702421:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967252864:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766741.702477:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766741.702504:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766741/real 1775766741] req@ffff8f9970a74700 x1862022967252864/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766741.702518:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967252864/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766741.702525:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766741.702528:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766741.702532:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967252864:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766741.702539:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967252992:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766741.702562:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766741.702574:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766741/real 1775766741] req@ffff8f9970a75c00 x1862022967252992/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766741.702587:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967252992/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766741.702593:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766741.702595:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766741.702625:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967252992:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766742.597870:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766742.597944:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766743.621846:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766743.621910:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766744.645822:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766744.645884:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766745.669914:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766745.670003:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766746.693851:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766746.693916:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766746.821950:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766746.821964:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766746.821971:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766746.821986:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3368 00000100:00080000:1.0:1775766746.822000:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766746.822009:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766746.822051:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967253120/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766746.822142:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766746.822147:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766746.822175:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766746.822181:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3368 00000100:00080000:1.0:1775766746.822187:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766746.822192:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766746.822204:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766746.822213:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967253120:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766746.822226:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967253248/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766746.822250:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766746.822271:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766746.822274:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766746.822278:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3368 00000100:00080000:1.0:1775766746.822283:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766746.822290:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766746.822299:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766746.822319:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967253376/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766746.822354:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1775766746.822358:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775766746.822364:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766746.822369:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766746.822377:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3368 00000100:00080000:1.0:1775766746.822408:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766746.822416:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766746.822425:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766746.822427:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766746/real 1775766746] req@ffff8f994f2e4700 x1862022967253120/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766746.822454:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967253504/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766746.822459:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967253120/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766746.822491:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766746.822497:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766746.822502:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967253120:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766746.822530:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967253248:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766746.822556:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766746.822569:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766746/real 1775766746] req@ffff8f994f2e6300 x1862022967253248/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766746.822582:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967253248/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766746.822590:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766746.822594:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766746.822597:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967253248:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766746.822642:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967253376:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766746.822714:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766746.822734:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766746/real 1775766746] req@ffff8f994f2e4e00 x1862022967253376/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766746.822791:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967253376/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766746.822824:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766746.822830:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766746.822836:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967253376:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766746.822844:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967253504:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766746.822876:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766746.822893:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766746/real 1775766746] req@ffff8f994f2e7b80 x1862022967253504/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766746.822914:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967253504/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766746.822928:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766746.822933:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766746.822939:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967253504:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766747.717842:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766747.717930:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766748.741867:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766748.741940:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766749.765829:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766749.765896:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766750.789819:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766750.789929:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766751.813885:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766751.814037:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766751.941913:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766751.941921:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766751.941925:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766751.941933:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3373 00000100:00080000:3.0:1775766751.941943:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766751.941948:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766751.941981:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967253632/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766751.942009:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766751.942012:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766751.942013:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766751.942017:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3373 00000100:00080000:3.0:1775766751.942019:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766751.942022:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766751.942030:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766751.942041:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967253760/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766751.942049:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766751.942051:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766751.942052:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766751.942056:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3373 00000100:00080000:3.0:1775766751.942058:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766751.942061:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766751.942064:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766751.942076:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967253888/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766751.942087:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967253632:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775766751.942091:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766751.942105:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766751.942110:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766751.942113:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3373 00000100:00080000:3.0:1775766751.942115:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766751.942117:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766751.942120:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766751.942127:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967254016/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766751.942226:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766751.942301:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766751/real 1775766751] req@ffff8f9970edfb80 x1862022967253632/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766751.942446:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967253632/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766751.942455:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766751.942458:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766751.942462:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967253632:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766751.942470:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967253760:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766751.942521:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766751.942540:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766751/real 1775766751] req@ffff8f9970edc380 x1862022967253760/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766751.942588:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967253760/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766751.942598:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766751.942601:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766751.942623:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967253760:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766751.942661:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967253888:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766751.942684:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766751.942694:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766751/real 1775766751] req@ffff8f9970edca80 x1862022967253888/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766751.942705:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967253888/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766751.942771:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766751.942775:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766751.942778:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967253888:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766751.942783:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967254016:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766751.942801:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766751.942811:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766751/real 1775766751] req@ffff8f9970edea00 x1862022967254016/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766751.942821:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967254016/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766751.942844:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766751.942846:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766751.942848:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967254016:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766752.837843:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766752.837919:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766753.861827:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766753.861867:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766754.885825:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766754.885906:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766755.909838:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766755.909931:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766756.933812:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766756.933881:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766757.061857:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766757.061866:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766757.061871:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766757.061880:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3378 00000100:00080000:1.0:1775766757.061891:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766757.061898:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766757.061940:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967254144/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766757.061977:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766757.061983:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766757.061985:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766757.062008:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3378 00000100:00080000:1.0:1775766757.062014:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766757.062020:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766757.062031:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766757.062056:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967254144:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766757.062117:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967254272/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766757.062132:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766757.062136:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766757.062139:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1775766757.062142:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775766757.062151:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3378 00000100:00080000:1.0:1775766757.062159:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766757.062166:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766757.062174:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766757.062192:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967254400/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766757.062200:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766757/real 1775766757] req@ffff8f994f2e7b80 x1862022967254144/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766757.062208:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766757.062216:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766757.062221:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766757.062227:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3378 00000100:00080000:1.0:1775766757.062233:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775766757.062233:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967254144/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766757.062242:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766757.062253:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1775766757.062255:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1775766757.062266:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967254528/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766757.062283:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766757.062288:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967254144:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766757.062311:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967254272:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766757.062332:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766757.062342:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766757/real 1775766757] req@ffff8f994f2e4e00 x1862022967254272/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766757.062352:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967254272/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766757.062359:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766757.062362:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766757.062365:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967254272:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766757.062370:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967254400:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766757.062383:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766757.062406:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766757/real 1775766757] req@ffff8f994f2e6300 x1862022967254400/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766757.062417:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967254400/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766757.062424:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766757.062426:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766757.062429:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967254400:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766757.062433:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967254528:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766757.062485:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766757.062512:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766757/real 1775766757] req@ffff8f994f2e4700 x1862022967254528/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766757.062524:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967254528/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766757.062532:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766757.062552:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766757.062554:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967254528:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766757.957910:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766757.957975:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766758.981830:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766758.981898:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766760.005810:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766760.005858:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766761.029828:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766761.029835:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:0.0:1775766762.053857:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766762.053929:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766762.181816:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766762.181825:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766762.181865:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766762.181874:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3383 00000100:00080000:1.0:1775766762.181886:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766762.181893:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766762.181927:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967254656/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766762.181958:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766762.181963:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766762.181965:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766762.181969:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3383 00000100:00080000:1.0:1775766762.181974:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766762.181978:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766762.181986:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766762.181992:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967254656:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766762.182001:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967254784/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766762.182011:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766762.182013:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766762.182015:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766762.182019:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3383 00000100:00080000:1.0:1775766762.182022:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766762.182025:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766762.182029:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766762.182037:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967254912/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766762.182056:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775766762.182066:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766762.182068:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766762.182070:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766762.182075:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3383 00000100:00080000:1.0:1775766762.182078:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766762.182081:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766762.182085:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766762.182097:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766762/real 1775766762] req@ffff8f994f2e6a00 x1862022967254656/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766762.182097:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967255040/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766762.182137:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967254656/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766762.182150:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766762.182155:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766762.182161:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967254656:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766762.182188:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967254784:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775766762.182215:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766762.182239:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766762/real 1775766762] req@ffff8f994f2e5f80 x1862022967254784/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766762.182252:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967254784/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766762.182260:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766762.182285:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766762.182289:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967254784:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775766762.182297:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967254912:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766762.182326:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766762.182344:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766762/real 1775766762] req@ffff8f994f2e4000 x1862022967254912/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766762.182354:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967254912/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766762.182360:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766762.182362:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766762.182364:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967254912:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766762.182368:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967255040:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766762.182382:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766762.182388:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766762/real 1775766762] req@ffff8f994f2e6680 x1862022967255040/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766762.182395:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967255040/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766762.182400:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766762.182402:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766762.182404:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967255040:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766763.077818:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766763.077873:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766764.101847:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766764.101919:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766765.125832:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766765.125877:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766766.149886:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766766.149886:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766767.173850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766767.173859:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766767.301816:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766767.301824:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766767.301829:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766767.301838:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3388 00000100:00080000:0.0:1775766767.301848:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766767.301853:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766767.301885:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967255168/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766767.301908:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766767.301911:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766767.301912:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766767.301915:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3388 00000100:00080000:0.0:1775766767.301917:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766767.301920:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766767.301928:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766767.301938:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967255296/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766767.301944:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967255168:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775766767.301948:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766767.301952:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766767.301953:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766767.301963:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3388 00000100:00080000:0.0:1775766767.301966:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766767.301972:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766767.301976:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766767.301984:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967255424/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766767.301992:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766767.301995:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766767.301996:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766767.301999:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3388 00000100:00080000:0.0:1775766767.302004:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775766767.302004:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775766767.302010:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766767.302013:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766767.302022:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967255552/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766767.302058:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766767/real 1775766767] req@ffff8f9970a77480 x1862022967255168/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766767.302075:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967255168/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766767.302083:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766767.302087:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766767.302089:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967255168:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766767.302107:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967255296:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766767.302161:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766767.302171:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766767/real 1775766767] req@ffff8f9970a77800 x1862022967255296/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766767.302179:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967255296/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766767.302184:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766767.302187:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766767.302189:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967255296:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766767.302193:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967255424:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766767.302205:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766767.302210:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766767/real 1775766767] req@ffff8f9970a76a00 x1862022967255424/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766767.302217:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967255424/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766767.302222:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766767.302224:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766767.302226:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967255424:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766767.302229:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967255552:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766767.302238:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766767.302244:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766767/real 1775766767] req@ffff8f9970a75500 x1862022967255552/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766767.302274:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967255552/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766767.302284:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766767.302287:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766767.302291:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967255552:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766768.197849:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766768.197912:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766769.221838:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766769.221898:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766770.245899:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766770.245925:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766771.269822:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766771.269844:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775766772.293918:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766772.293918:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00080000:3.0:1775766772.421903:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766772.421917:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766772.421923:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766772.421938:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3394 00000100:00080000:3.0:1775766772.421955:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766772.421965:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766772.422023:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddf80 x1862022967255680/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766772.422060:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766772.422066:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766772.422078:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766772.422085:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3394 00000100:00080000:3.0:1775766772.422090:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766772.422095:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766772.422103:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967255680:192.168.202.146@tcp:250:kworker.0 00000008:00100000:3.0:1775766772.422107:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766772.422162:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd180 x1862022967255808/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766772.422180:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766772.422186:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766772.422189:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766772.422194:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3394 00000100:00080000:3.0:1775766772.422210:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766772.422216:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766772.422223:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775766772.422232:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766772.422254:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967255936/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766772.422270:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766772.422275:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766772.422277:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766772.422281:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3394 00000100:00080000:3.0:1775766772.422286:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766772.422290:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766772.422297:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766772.422309:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967256064/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766772.422390:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766772/real 1775766772] req@ffff8f9970eddf80 x1862022967255680/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766772.422414:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddf80 x1862022967255680/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766772.422425:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766772.422429:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766772.422432:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967255680:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766772.422470:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967255808:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766772.422492:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766772.422502:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766772/real 1775766772] req@ffff8f9970edd180 x1862022967255808/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766772.422511:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd180 x1862022967255808/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766772.422658:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766772.422667:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766772.422672:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967255808:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766772.422698:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967255936:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766772.422856:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766772.422894:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766772/real 1775766772] req@ffff8f9970edea00 x1862022967255936/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766772.422915:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967255936/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766772.422926:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766772.422931:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766772.422935:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967255936:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766772.422943:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967256064:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766772.422965:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766772.422977:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766772/real 1775766772] req@ffff8f9970edca80 x1862022967256064/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766772.422991:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967256064/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766772.423001:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766772.423005:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766772.423045:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967256064:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766773.317822:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766773.317956:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766774.341869:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766774.341910:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766775.365830:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766775.365891:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766776.389832:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766776.389858:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766777.413857:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766777.413918:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766777.541865:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766777.541875:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766777.541880:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766777.541890:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3399 00000100:00080000:1.0:1775766777.541911:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766777.541918:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766777.541958:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967256192/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766777.541987:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775766777.542016:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967256192:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775766777.542033:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766777.542035:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766777.542041:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3399 00000100:00080000:1.0:1775766777.542046:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766777.542051:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766777.542060:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766777.542076:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967256320/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766777.542090:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766777.542093:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766777.542095:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766777.542100:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3399 00000100:00080000:1.0:1775766777.542105:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775766777.542109:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967256320:192.168.202.146@tcp:38:kworker.0 00000100:00080000:1.0:1775766777.542111:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766777.542123:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766777.542140:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967256448/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766777.542159:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766777.542163:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766777.542165:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766777.542170:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3399 00000100:00080000:1.0:1775766777.542176:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775766777.542178:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967256448:192.168.202.146@tcp:8:kworker.0 00000100:00080000:1.0:1775766777.542184:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766777.542207:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766777.542223:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967256576/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766777.542262:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967256576:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766778.437830:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766778.437900:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766779.461833:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766779.461900:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766780.485847:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766780.485933:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1775766780.614165:0:2367:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:0.0:1775766780.614232:0:2367:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1775766780.614310:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775766780.614352:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775766780.614362:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775766780.614374:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:2.0:1775766780.614403:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766777/real 1775766780] req@ffff8f994f2e6d80 x1862022967256576/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766780.614433:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967256576/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766780.614446:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766780.614452:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766780.614456:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967256576:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766780.614465:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766777/real 1775766780] req@ffff8f994f2e4a80 x1862022967256448/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766780.614478:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967256448/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766780.614486:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766780.614490:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766780.614529:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967256448:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766780.614536:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766777/real 1775766780] req@ffff8f994f2e7800 x1862022967256320/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766780.614549:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967256320/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766780.614556:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766780.614560:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766780.614563:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967256320:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766780.614569:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766777/real 1775766780] req@ffff8f994f2e5180 x1862022967256192/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766780.614582:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967256192/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766780.614589:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766780.614593:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766780.614597:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967256192:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766781.509867:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766781.509901:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766782.533842:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766782.533925:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766782.661893:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766782.661919:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766782.661925:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766782.661940:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3404 00000100:00080000:1.0:1775766782.661956:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766782.661964:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766782.662021:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967256704/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766782.662052:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766782.662055:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766782.662056:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766782.662060:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3404 00000100:00080000:1.0:1775766782.662063:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766782.662066:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766782.662076:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766782.662088:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967256832/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766782.662096:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766782.662101:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1775766782.662129:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967256704:192.168.202.146@tcp:250:kworker.0 00000800:00000100:2.0:1775766782.662271:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775766782.662273:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766782.662284:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3404 00000100:00080000:1.0:1775766782.662292:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766782.662301:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766782.662316:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766782.662384:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766782/real 1775766782] req@ffff8f994f2e5180 x1862022967256704/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766782.662391:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967256960/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766782.662433:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967256704/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766782.662436:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766782.662447:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766782.662452:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766782.662460:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3404 00000100:00080000:1.0:1775766782.662468:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766782.662472:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766782.662480:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766782.662495:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967257088/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766782.662527:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766782.662536:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766782.662542:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967256704:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766782.662555:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967256832:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766782.662595:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766782.662615:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766782/real 1775766782] req@ffff8f994f2e7800 x1862022967256832/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766782.662632:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967256832/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766782.662643:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766782.662648:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766782.662652:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967256832:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766782.662691:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967256960:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766782.662725:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766782.662780:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766782/real 1775766782] req@ffff8f994f2e4a80 x1862022967256960/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766782.662798:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967256960/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766782.662810:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766782.662815:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766782.662819:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967256960:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766782.662851:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967257088:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766782.662884:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766782.662903:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766782/real 1775766782] req@ffff8f994f2e6d80 x1862022967257088/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766782.662918:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967257088/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766782.662928:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766782.662933:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766782.662937:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967257088:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766783.557863:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766783.557966:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766784.581810:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766784.581889:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766785.605826:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766785.605921:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766786.629810:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766786.629836:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766787.653958:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766787.654015:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766787.781898:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766787.781909:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766787.781914:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766787.781926:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3409 00000100:00080000:3.0:1775766787.781940:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766787.781948:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766787.781998:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967257216/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766787.782028:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766787.782031:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766787.782033:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766787.782037:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3409 00000100:00080000:3.0:1775766787.782040:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766787.782043:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766787.782053:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766787.782064:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967257344/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766787.782074:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766787.782076:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766787.782080:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775766787.782082:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967257216:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775766787.782085:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3409 00000100:00080000:3.0:1775766787.782092:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766787.782096:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766787.782100:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766787.782113:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967257472/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766787.782126:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766787.782129:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766787.782130:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766787.782133:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3409 00000100:00080000:3.0:1775766787.782135:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766787.782137:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766787.782141:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766787.782149:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967257600/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766787.782193:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766787.782249:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766787/real 1775766787] req@ffff8f9970edc380 x1862022967257216/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766787.782269:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967257216/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766787.782279:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766787.782283:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766787.782287:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967257216:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766787.782312:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967257344:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775766787.782535:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766787.782563:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766787/real 1775766787] req@ffff8f9970edfb80 x1862022967257344/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766787.782580:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967257344/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766787.782590:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766787.782593:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766787.782596:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967257344:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766787.782606:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967257472:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766787.782621:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766787.782628:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766787/real 1775766787] req@ffff8f9970ede300 x1862022967257472/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766787.782637:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967257472/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766787.782642:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766787.782645:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766787.782647:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967257472:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766787.782651:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967257600:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775766787.782662:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766787.782668:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766787/real 1775766787] req@ffff8f9970edd880 x1862022967257600/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766787.782707:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967257600/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766787.782714:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766787.782717:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775766787.782719:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967257600:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766788.677874:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766788.677980:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766789.701843:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766789.701929:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766790.725846:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766790.725980:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766791.749818:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766791.749833:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766792.773924:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766792.773964:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766792.901859:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766792.901868:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766792.901873:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766792.901881:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3414 00000100:00080000:2.0:1775766792.901892:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766792.901897:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766792.901932:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0380 x1862022967257728/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766792.901961:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766792.901964:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766792.901965:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766792.901969:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3414 00000100:00080000:2.0:1775766792.901971:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766792.901974:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766792.901982:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766792.902028:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0000 x1862022967257856/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766792.902037:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766792.902040:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766792.902042:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766792.902047:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3414 00000100:00080000:2.0:1775766792.902051:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775766792.902052:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967257728:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775766792.902054:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766792.902062:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766792.902069:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2680 x1862022967257984/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766792.902076:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766792.902079:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766792.902080:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766792.902083:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3414 00000100:00080000:2.0:1775766792.902085:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766792.902087:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766792.902091:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766792.902099:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3800 x1862022967258112/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766792.902140:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766792.902182:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766792/real 1775766792] req@ffff8f99703d0380 x1862022967257728/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766792.902199:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0380 x1862022967257728/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766792.902208:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766792.902211:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766792.902214:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967257728:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766792.902234:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967257856:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775766792.902273:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766792.902288:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766792/real 1775766792] req@ffff8f99703d0000 x1862022967257856/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766792.902321:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0000 x1862022967257856/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766792.902332:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766792.902336:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766792.902341:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967257856:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775766792.902348:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967257984:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766792.902388:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766792.902397:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766792/real 1775766792] req@ffff8f99703d2680 x1862022967257984/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766792.902406:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2680 x1862022967257984/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766792.902413:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766792.902415:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766792.902417:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967257984:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766792.902421:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967258112:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766792.902434:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766792.902440:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766792/real 1775766792] req@ffff8f99703d3800 x1862022967258112/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766792.902453:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3800 x1862022967258112/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766792.902458:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766792.902460:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766792.902462:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967258112:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766793.797871:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766793.798010:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766794.821825:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766794.821873:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766795.845877:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766795.845985:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766796.869909:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766796.869922:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766797.893876:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766797.894019:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766798.021874:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766798.021887:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766798.021945:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766798.021964:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3419 00000100:00080000:2.0:1775766798.021983:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766798.021997:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766798.022056:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3800 x1862022967258240/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766798.022143:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766798.022168:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766798.022172:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766798.022182:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3419 00000100:00080000:2.0:1775766798.022195:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766798.022205:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766798.022239:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766798.022251:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967258240:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766798.022275:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2680 x1862022967258368/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766798.022301:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766798.022308:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766798.022313:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766798.022341:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3419 00000800:00000100:3.0:1775766798.022343:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775766798.022357:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766798.022367:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766798.022377:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766798.022401:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0000 x1862022967258496/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766798.022411:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766798/real 1775766798] req@ffff8f99703d3800 x1862022967258240/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766798.022535:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766798.022549:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766798.022556:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766798.022571:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3419 00000100:00080000:2.0:1775766798.022580:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766798.022589:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766798.022597:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766798.022614:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0380 x1862022967258624/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766798.022792:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3800 x1862022967258240/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766798.022812:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766798.022934:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766798.022953:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967258240:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766798.022985:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967258368:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766798.023030:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766798.023050:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766798/real 1775766798] req@ffff8f99703d2680 x1862022967258368/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766798.023098:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2680 x1862022967258368/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766798.023110:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766798.023115:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766798.023120:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967258368:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766798.023129:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967258496:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766798.023159:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766798.023174:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766798/real 1775766798] req@ffff8f99703d0000 x1862022967258496/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766798.023189:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0000 x1862022967258496/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766798.023199:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766798.023203:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766798.023207:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967258496:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766798.023213:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967258624:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766798.023232:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766798.023244:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766798/real 1775766798] req@ffff8f99703d0380 x1862022967258624/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766798.023257:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0380 x1862022967258624/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766798.023268:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766798.023272:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766798.023275:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967258624:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766798.917903:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766798.918088:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766799.941898:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766799.942105:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766800.965871:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766800.966018:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766801.989810:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766801.989930:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766803.013827:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766803.013878:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766803.142410:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766803.142433:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766803.142440:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766803.142455:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3424 00000100:00080000:3.0:1775766803.142472:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766803.142482:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766803.142530:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967258752/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766803.142563:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766803.142566:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766803.142568:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766803.142572:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3424 00000100:00080000:3.0:1775766803.142576:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766803.142579:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766803.142589:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766803.142600:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967258880/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766803.142608:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967258752:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775766803.142611:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766803.142617:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766803.142619:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766803.142623:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3424 00000100:00080000:3.0:1775766803.142625:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766803.142628:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766803.142632:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766803.142640:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967259008/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766803.142647:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766803.142649:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766803.142651:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766803.142654:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3424 00000100:00080000:3.0:1775766803.142656:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766803.142658:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766803.142661:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766803.142669:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967259136/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766803.142679:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766803.142723:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766803/real 1775766803] req@ffff8f9970edf480 x1862022967258752/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766803.142770:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967258752/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766803.142780:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766803.142783:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766803.142796:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967258752:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766803.142822:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967258880:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775766803.142847:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766803.142861:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766803/real 1775766803] req@ffff8f9970edf800 x1862022967258880/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766803.142872:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967258880/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766803.142878:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766803.142880:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766803.142883:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967258880:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775766803.142887:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967259008:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766803.142902:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766803.142909:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766803/real 1775766803] req@ffff8f9970edc000 x1862022967259008/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766803.142916:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967259008/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766803.142921:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766803.142923:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766803.142925:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967259008:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766803.142929:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967259136:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766803.142938:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766803.142944:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766803/real 1775766803] req@ffff8f9970ede680 x1862022967259136/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766803.142974:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967259136/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766803.142980:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766803.142983:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766803.142987:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967259136:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766804.037880:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766804.038029:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766805.061898:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766805.061930:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766806.085849:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766806.085898:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766807.110029:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766807.110054:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766808.133922:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766808.134058:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766808.261930:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766808.261949:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766808.261957:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766808.261992:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3429 00000100:00080000:0.0:1775766808.262012:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766808.262023:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766808.262099:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967259264/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766808.262158:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766808.262163:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766808.262165:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766808.262171:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3429 00000100:00080000:0.0:1775766808.262176:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766808.262183:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766808.262198:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766808.262211:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967259264:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766808.262217:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967259392/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766808.262234:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766808.262238:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766808.262240:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766808.262244:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3429 00000100:00080000:0.0:1775766808.262248:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766808.262252:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766808.262258:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766808.262273:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967259520/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766808.262284:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766808.262291:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766808.262294:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766808.262300:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3429 00000100:00080000:0.0:1775766808.262307:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1775766808.262314:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775766808.262320:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766808.262331:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766808.262359:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967259648/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766808.262380:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766808/real 1775766808] req@ffff8f9970a75180 x1862022967259264/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766808.262405:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967259264/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766808.262424:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766808.262447:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766808.262455:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967259264:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766808.262496:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967259392:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766808.262538:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766808.262556:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766808/real 1775766808] req@ffff8f9970a76300 x1862022967259392/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766808.262571:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967259392/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766808.262581:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766808.262584:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766808.262589:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967259392:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766808.262596:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967259520:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766808.262632:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766808.262644:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766808/real 1775766808] req@ffff8f9970a77100 x1862022967259520/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766808.262657:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967259520/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766808.262666:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766808.262669:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766808.262672:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967259520:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766808.262678:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967259648:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766808.262697:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766808.262707:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766808/real 1775766808] req@ffff8f9970a75500 x1862022967259648/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766808.262719:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967259648/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766808.262727:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766808.262730:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766808.262733:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967259648:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766809.157834:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766809.157902:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766810.181852:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766810.181902:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766811.205980:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766811.206077:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766812.229912:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766812.230137:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766813.253892:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766813.253970:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766813.381826:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766813.381836:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766813.381858:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766813.381869:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3434 00000100:00080000:0.0:1775766813.381881:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766813.381888:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766813.381960:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967259776/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766813.381989:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766813.381994:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766813.381995:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766813.382001:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3434 00000100:00080000:0.0:1775766813.382005:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766813.382011:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766813.382021:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766813.382023:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967259776:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766813.382037:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967259904/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766813.382050:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766813.382054:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766813.382056:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766813.382060:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3434 00000100:00080000:0.0:1775766813.382065:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766813.382069:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766813.382075:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766813.382083:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766813.382089:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967260032/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766813.382117:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766813.382121:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766813.382124:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775766813.382127:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766813/real 1775766813] req@ffff8f9970a76a00 x1862022967259776/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766813.382131:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3434 00000100:00080000:0.0:1775766813.382140:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766813.382146:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766813.382151:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967259776/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1775766813.382153:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1775766813.382164:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766813.382175:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766813.382180:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967260160/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766813.382180:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967259776:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766813.382204:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967259904:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766813.382221:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766813.382229:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766813/real 1775766813] req@ffff8f9970a77800 x1862022967259904/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766813.382241:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967259904/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766813.382247:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766813.382248:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766813.382250:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967259904:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766813.382254:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967260032:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766813.382264:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766813.382270:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766813/real 1775766813] req@ffff8f9970a77480 x1862022967260032/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766813.382277:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967260032/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766813.382282:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766813.382283:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766813.382285:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967260032:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766813.382288:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967260160:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766813.382297:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766813.382303:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766813/real 1775766813] req@ffff8f9970a75c00 x1862022967260160/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766813.382309:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967260160/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766813.382328:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766813.382330:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766813.382332:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967260160:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766814.277888:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766814.278093:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766815.301879:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766815.301985:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766816.325833:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766816.325923:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766817.349960:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766817.349964:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766818.373930:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766818.373932:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00080000:0.0:1775766818.501822:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766818.501831:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766818.501836:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766818.501845:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3440 00000100:00080000:0.0:1775766818.501869:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766818.501876:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766818.501911:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967260288/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766818.501978:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766818.501982:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766818.501984:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766818.501989:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3440 00000100:00080000:0.0:1775766818.501993:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766818.501998:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766818.502007:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766818.502008:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967260288:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766818.502027:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967260416/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766818.502039:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766818.502044:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766818.502046:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766818.502050:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3440 00000100:00080000:0.0:1775766818.502055:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766818.502060:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766818.502066:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766818.502070:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766818.502086:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967260544/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766818.502118:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766818.502123:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766818.502126:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766818.502134:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3440 00000100:00100000:3.0:1775766818.502139:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766818/real 1775766818] req@ffff8f9970a74700 x1862022967260288/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766818.502180:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967260288/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766818.502185:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766818.502200:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766818.502201:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00080000:3.0:1775766818.502210:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766818.502215:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967260288:192.168.202.146@tcp:250:kworker.0 00000008:00100000:0.0:1775766818.502216:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766818.502243:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967260416:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775766818.502260:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967260672/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766818.502275:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766818.502297:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766818/real 1775766818] req@ffff8f9970a76d80 x1862022967260416/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766818.502329:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967260416/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766818.502337:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766818.502342:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766818.502346:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967260416:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766818.502353:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967260544:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766818.502374:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766818.502384:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766818/real 1775766818] req@ffff8f9970a74380 x1862022967260544/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766818.502393:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967260544/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766818.502399:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766818.502401:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766818.502403:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967260544:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766818.502414:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967260672:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766818.502426:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766818.502432:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766818/real 1775766818] req@ffff8f9970a74700 x1862022967260672/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766818.502440:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967260672/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766818.502445:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766818.502446:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766818.502448:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967260672:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766819.397824:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766819.397930:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766820.421812:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766820.421884:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766821.445883:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766821.445938:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766822.469920:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766822.469995:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766823.493870:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766823.493887:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00080000:3.0:1775766823.621909:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766823.621918:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766823.621922:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766823.621932:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3445 00000100:00080000:3.0:1775766823.621942:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766823.621949:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766823.621985:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967260800/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766823.622008:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766823.622011:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766823.622012:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766823.622016:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3445 00000100:00080000:3.0:1775766823.622018:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766823.622020:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766823.622028:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766823.622038:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967260928/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766823.622045:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766823.622048:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766823.622049:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766823.622051:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3445 00000100:00080000:3.0:1775766823.622053:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766823.622055:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766823.622059:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766823.622066:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967260800:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766823.622071:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967261056/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766823.622086:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766823.622088:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766823.622089:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766823.622091:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3445 00000100:00080000:3.0:1775766823.622093:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766823.622096:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766823.622098:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766823.622106:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967261184/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766823.622171:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766823.622236:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766823/real 1775766823] req@ffff8f9970edd500 x1862022967260800/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766823.622260:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967260800/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766823.622273:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766823.622278:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766823.622283:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967260800:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766823.622293:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967260928:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775766823.622319:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766823.622335:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766823/real 1775766823] req@ffff8f9970eded80 x1862022967260928/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766823.622348:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967260928/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766823.622357:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766823.622361:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766823.622366:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967260928:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775766823.622399:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967261056:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766823.622423:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766823.622435:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766823/real 1775766823] req@ffff8f9970edce00 x1862022967261056/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766823.622459:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967261056/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766823.622468:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766823.622472:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766823.622476:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967261056:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766823.622483:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967261184:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766823.622502:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766823.622513:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766823/real 1775766823] req@ffff8f9970edc700 x1862022967261184/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766823.622527:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967261184/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766823.622536:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766823.622539:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766823.622543:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967261184:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766824.517852:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766824.517974:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766825.541909:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766825.541933:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766826.565828:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766826.565908:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766827.589829:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766827.589891:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766828.613831:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766828.613899:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766828.741900:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766828.741911:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766828.741917:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766828.741927:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3450 00000100:00080000:0.0:1775766828.741939:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766828.741945:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766828.742004:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967261312/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766828.742027:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766828.742030:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766828.742031:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766828.742035:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3450 00000100:00080000:0.0:1775766828.742038:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766828.742057:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766828.742065:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766828.742075:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967261440/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766828.742084:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766828.742088:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1775766828.742088:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967261312:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775766828.742090:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766828.742098:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3450 00000100:00080000:0.0:1775766828.742100:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766828.742103:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766828.742106:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766828.742116:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967261568/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766828.742134:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766828.742138:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766828.742140:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766828.742145:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3450 00000100:00080000:0.0:1775766828.742150:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766828.742155:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:3.0:1775766828.742160:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:0.0:1775766828.742164:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766828.742181:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967261696/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766828.742223:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766828/real 1775766828] req@ffff8f9970a75c00 x1862022967261312/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766828.742245:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967261312/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766828.742257:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766828.742261:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766828.742266:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967261312:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766828.742304:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967261440:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766828.742335:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766828.742350:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766828/real 1775766828] req@ffff8f9970a77480 x1862022967261440/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766828.742365:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967261440/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766828.742374:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766828.742378:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766828.742381:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967261440:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766828.742386:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967261568:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766828.742403:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766828.742411:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766828/real 1775766828] req@ffff8f9970a77800 x1862022967261568/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766828.742419:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967261568/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766828.742425:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766828.742428:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766828.742430:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967261568:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766828.742433:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967261696:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766828.742444:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766828.742450:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766828/real 1775766828] req@ffff8f9970a76a00 x1862022967261696/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766828.742463:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967261696/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766828.742468:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766828.742470:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766828.742472:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967261696:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766829.637841:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766829.637875:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766830.661826:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766830.661882:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766831.685828:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766831.685889:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766832.709890:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766832.709925:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775766833.733884:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766833.733918:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766833.861835:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766833.861849:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766833.861862:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766833.861876:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3455 00000100:00080000:0.0:1775766833.861891:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766833.861899:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766833.861940:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967261824/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766833.861969:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766833.861973:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766833.861975:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766833.861980:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3455 00000100:00080000:0.0:1775766833.861984:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766833.861988:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766833.862005:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967261824:192.168.202.146@tcp:250:kworker.0 00000800:00000100:3.0:1775766833.862092:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:0.0:1775766833.862114:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766833.862150:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967261952/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766833.862171:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766833.862177:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766833.862180:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766833.862187:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3455 00000100:00080000:0.0:1775766833.862198:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766833.862205:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766833.862211:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766833.862221:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967262080/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766833.862229:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766833.862231:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766833.862232:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766833.862235:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3455 00000100:00080000:0.0:1775766833.862237:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766833.862240:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766833.862242:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766833.862249:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967262208/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766833.862343:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766833/real 1775766833] req@ffff8f9970a75500 x1862022967261824/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766833.862418:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967261824/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766833.862430:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766833.862436:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766833.862444:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967261824:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766833.862474:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967261952:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766833.862496:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766833.862505:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766833/real 1775766833] req@ffff8f9970a77100 x1862022967261952/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766833.862514:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967261952/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766833.862524:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766833.862527:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766833.862529:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967261952:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766833.862533:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967262080:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766833.862545:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766833.862550:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766833/real 1775766833] req@ffff8f9970a76300 x1862022967262080/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766833.862558:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967262080/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766833.862564:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766833.862565:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766833.862568:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967262080:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766833.862571:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967262208:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766833.862581:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766833.862598:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766833/real 1775766833] req@ffff8f9970a75180 x1862022967262208/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766833.862605:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967262208/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766833.862610:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766833.862612:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766833.862615:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967262208:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766834.757819:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766834.757877:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766835.781827:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766835.781892:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766836.805795:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766836.805863:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766837.829831:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766837.829902:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766838.853816:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766838.853833:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766838.981833:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766838.981840:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766838.981844:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766838.981853:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3460 00000100:00080000:0.0:1775766838.981862:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766838.981867:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766838.981899:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967262336/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766838.981933:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766838.981935:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766838.981937:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766838.981940:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3460 00000100:00080000:0.0:1775766838.981942:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766838.981945:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766838.981952:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766838.981961:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967262464/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766838.981969:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766838.981971:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766838.981973:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775766838.981977:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967262336:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775766838.981978:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3460 00000100:00080000:0.0:1775766838.981982:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766838.981987:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766838.981990:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766838.981998:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967262592/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766838.982010:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766838.982012:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766838.982014:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766838.982016:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3460 00000100:00080000:0.0:1775766838.982019:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766838.982021:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766838.982023:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766838.982031:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967262720/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766838.982039:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766838.982086:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766838/real 1775766838] req@ffff8f9970a75180 x1862022967262336/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766838.982101:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967262336/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766838.982110:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766838.982114:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766838.982128:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967262336:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766838.982146:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967262464:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766838.982162:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766838.982170:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766838/real 1775766838] req@ffff8f9970a76300 x1862022967262464/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766838.982178:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967262464/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766838.982184:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766838.982186:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766838.982188:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967262464:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766838.982192:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967262592:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766838.982205:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766838.982211:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766838/real 1775766838] req@ffff8f9970a77100 x1862022967262592/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766838.982219:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967262592/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766838.982223:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766838.982225:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766838.982227:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967262592:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766838.982231:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967262720:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766838.982240:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766838.982246:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766838/real 1775766838] req@ffff8f9970a75500 x1862022967262720/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766838.982253:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967262720/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766838.982262:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766838.982265:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766838.982268:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967262720:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766839.877895:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766839.877896:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766840.901832:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766840.901875:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766841.925849:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766841.925917:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766842.949856:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766842.949913:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766843.973854:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766843.973872:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766844.101880:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766844.101892:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766844.101897:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766844.101910:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3465 00000100:00080000:0.0:1775766844.101924:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766844.101932:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766844.101983:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967262848/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766844.102012:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766844.102016:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766844.102018:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766844.102022:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3465 00000100:00080000:0.0:1775766844.102024:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766844.102027:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766844.102039:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766844.102053:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967262976/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766844.102074:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766844.102077:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766844.102079:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766844.102081:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3465 00000100:00080000:0.0:1775766844.102084:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766844.102088:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766844.102092:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766844.102104:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967263104/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766844.102109:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967262848:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775766844.102121:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766844.102129:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766844.102130:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766844.102134:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3465 00000100:00080000:0.0:1775766844.102137:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766844.102139:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766844.102143:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766844.102155:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967263232/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766844.102264:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967262976:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766844.102309:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967263104:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766844.102362:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967263232:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766844.997852:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766844.997943:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766846.021849:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766846.021940:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766847.045871:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766847.045887:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000400:00000100:2.0:1775766847.174111:0:2369:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:2.0:1775766847.174170:0:2369:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775766847.174190:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:2.0:1775766847.174244:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:2.0:1775766847.174255:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:2.0:1775766847.174265:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:3.0:1775766847.174344:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766844/real 1775766847] req@ffff8f9970a77480 x1862022967263104/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766847.174380:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967263104/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766847.174396:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766847.174403:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766847.174408:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967263104:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766847.174417:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766844/real 1775766847] req@ffff8f9970a75c00 x1862022967263232/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766847.174432:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967263232/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766847.174442:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766847.174447:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766847.174451:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967263232:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766847.174459:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766844/real 1775766847] req@ffff8f9970a76a00 x1862022967262848/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766847.174475:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967262848/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766847.174485:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766847.174489:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766847.174495:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967262848:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766847.174505:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766844/real 1775766847] req@ffff8f9970a77800 x1862022967262976/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766847.174519:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967262976/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766847.174529:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766847.174534:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766847.174538:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967262976:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775766848.069810:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766848.069895:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766849.093858:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766849.093895:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766849.221885:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766849.221895:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766849.221902:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766849.221915:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3470 00000100:00080000:0.0:1775766849.221928:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766849.221938:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766849.221990:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967263360/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766849.222033:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766849.222039:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766849.222042:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766849.222049:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3470 00000100:00080000:0.0:1775766849.222055:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766849.222062:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766849.222074:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766849.222090:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967263488/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766849.222100:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766849.222104:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766849.222106:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766849.222110:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3470 00000100:00080000:0.0:1775766849.222114:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775766849.222121:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967263360:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775766849.222122:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766849.222132:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766849.222142:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967263616/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766849.222160:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766849.222164:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766849.222166:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766849.222170:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3470 00000100:00080000:0.0:1775766849.222174:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766849.222177:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766849.222181:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766849.222193:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967263744/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766849.222226:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766849.222338:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766849/real 1775766849] req@ffff8f9970a74700 x1862022967263360/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766849.222366:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967263360/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766849.222392:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766849.222399:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766849.222405:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967263360:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766849.222417:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967263488:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766849.222456:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766849.222505:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766849/real 1775766849] req@ffff8f9970a74380 x1862022967263488/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766849.222574:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967263488/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766849.222587:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766849.222592:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766849.222597:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967263488:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766849.222632:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967263616:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766849.222664:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766849.222693:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766849/real 1775766849] req@ffff8f9970a76d80 x1862022967263616/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766849.222710:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967263616/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766849.222720:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766849.222725:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766849.222729:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967263616:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766849.222905:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967263744:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766849.222948:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766849.222969:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766849/real 1775766849] req@ffff8f9970a77800 x1862022967263744/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766849.222989:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967263744/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766849.223011:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766849.223016:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766849.223021:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967263744:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766850.117846:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766850.117909:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766851.141826:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766851.141958:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766852.165824:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766852.165887:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766853.189828:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766853.189877:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766854.213855:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766854.213904:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00080000:0.0:1775766854.341808:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766854.341818:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766854.341822:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766854.341833:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3475 00000100:00080000:0.0:1775766854.341844:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766854.341852:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766854.341906:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967263872/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766854.341953:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766854.341957:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766854.341959:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766854.341964:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3475 00000100:00080000:0.0:1775766854.341969:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766854.341974:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766854.341986:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766854.341994:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967263872:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766854.342003:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967264000/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766854.342013:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766854.342015:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766854.342017:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766854.342019:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3475 00000100:00080000:0.0:1775766854.342023:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766854.342026:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766854.342030:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766854.342038:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967264128/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766854.342045:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766854.342047:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766854.342048:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766854.342050:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3475 00000800:00000100:3.0:1775766854.342055:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775766854.342057:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766854.342061:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766854.342064:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766854.342072:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967264256/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766854.342099:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766854/real 1775766854] req@ffff8f9970a76a00 x1862022967263872/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766854.342113:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967263872/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766854.342120:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766854.342124:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766854.342126:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967263872:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766854.342145:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967264000:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766854.342161:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766854.342169:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766854/real 1775766854] req@ffff8f9970a75c00 x1862022967264000/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766854.342177:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967264000/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766854.342182:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766854.342184:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766854.342187:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967264000:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766854.342191:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967264128:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766854.342202:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766854.342207:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766854/real 1775766854] req@ffff8f9970a77480 x1862022967264128/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766854.342214:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967264128/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766854.342219:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766854.342221:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766854.342223:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967264128:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766854.342226:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967264256:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766854.342236:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766854.342241:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766854/real 1775766854] req@ffff8f9970a75500 x1862022967264256/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766854.342248:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967264256/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766854.342253:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766854.342254:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766854.342256:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967264256:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766855.237825:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766855.237888:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766856.261815:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766856.261873:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766857.285862:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766857.285900:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766858.309838:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766858.309906:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766859.333852:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766859.333882:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766859.461834:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766859.461842:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766859.461847:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766859.461857:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3481 00000100:00080000:0.0:1775766859.461869:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766859.461876:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766859.461927:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967264384/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766859.461953:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766859.461957:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766859.461959:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766859.461965:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3481 00000100:00080000:0.0:1775766859.461970:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766859.461975:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766859.461984:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766859.461999:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967264512/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766859.462002:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967264384:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775766859.462011:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766859.462014:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766859.462016:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766859.462020:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3481 00000100:00080000:0.0:1775766859.462023:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766859.462027:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766859.462032:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766859.462040:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967264640/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766859.462065:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775766859.462098:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766859.462101:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766859.462103:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766859.462108:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3481 00000100:00080000:0.0:1775766859.462112:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775766859.462113:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766859/real 1775766859] req@ffff8f9970a77100 x1862022967264384/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766859.462115:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766859.462121:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766859.462134:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967264384/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766859.462135:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967264768/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766859.462147:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766859.462152:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766859.462156:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967264384:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766859.462181:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967264512:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766859.462206:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766859.462220:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766859/real 1775766859] req@ffff8f9970a76300 x1862022967264512/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766859.462229:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967264512/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766859.462235:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766859.462238:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766859.462240:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967264512:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766859.462245:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967264640:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766859.462259:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766859.462265:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766859/real 1775766859] req@ffff8f9970a75180 x1862022967264640/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766859.462278:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967264640/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766859.462283:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766859.462285:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766859.462287:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967264640:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766859.462290:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967264768:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766859.462301:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766859.462324:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766859/real 1775766859] req@ffff8f9970a75500 x1862022967264768/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766859.462332:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967264768/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766859.462338:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766859.462340:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766859.462342:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967264768:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766860.357850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766860.358007:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766861.381905:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766861.381921:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775766862.405894:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766862.406044:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766863.429848:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766863.429909:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766864.453830:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766864.453860:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766864.581856:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766864.581863:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766864.581867:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766864.581876:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3486 00000100:00080000:3.0:1775766864.581885:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766864.581891:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766864.581924:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf100 x1862022967264896/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766864.581947:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766864.581951:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766864.581952:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766864.581956:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3486 00000100:00080000:3.0:1775766864.581959:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766864.581963:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766864.581971:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766864.581981:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967264896:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766864.581982:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddc00 x1862022967265024/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766864.581995:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766864.581997:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766864.581998:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766864.582001:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3486 00000100:00080000:3.0:1775766864.582003:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766864.582005:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766864.582008:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766864.582017:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967265152/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766864.582025:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766864.582027:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766864.582028:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766864.582031:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3486 00000100:00080000:3.0:1775766864.582035:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766864.582037:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766864.582040:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766864.582045:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766864.582049:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967265280/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766864.582101:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766864/real 1775766864] req@ffff8f9970edf100 x1862022967264896/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766889 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766864.582117:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf100 x1862022967264896/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766889 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766864.582125:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766864.582128:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766864.582131:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967264896:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766864.582169:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967265024:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775766864.582184:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766864.582192:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766864/real 1775766864] req@ffff8f9970eddc00 x1862022967265024/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766889 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766864.582200:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddc00 x1862022967265024/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766889 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766864.582206:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766864.582208:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766864.582210:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967265024:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775766864.582213:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967265152:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766864.582224:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766864.582240:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766864/real 1775766864] req@ffff8f9970edc700 x1862022967265152/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766889 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766864.582247:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967265152/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766889 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766864.582252:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766864.582254:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766864.582255:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967265152:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766864.582259:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967265280:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766864.582268:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766864.582273:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766864/real 1775766864] req@ffff8f9970edce00 x1862022967265280/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766889 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766864.582280:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967265280/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766889 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766864.582289:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766864.582292:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766864.582294:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967265280:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766865.477838:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766865.477873:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766866.501819:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766866.501876:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766867.525825:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766867.525917:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766868.549874:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766868.549892:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766869.573825:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766869.573830:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766869.702308:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766869.702329:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766869.702334:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766869.702345:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3491 00000100:00080000:3.0:1775766869.702356:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766869.702364:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766869.702406:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967265408/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766869.702437:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766869.702442:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766869.702445:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766869.702450:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3491 00000100:00080000:3.0:1775766869.702455:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766869.702460:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766869.702475:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766869.702490:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967265408:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766869.702492:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967265536/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766869.702520:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766869.702524:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766869.702526:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766869.702530:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3491 00000100:00080000:3.0:1775766869.702535:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766869.702539:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766869.702544:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766869.702558:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967265664/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766869.702565:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775766869.702574:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766869.702578:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766869.702580:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766869.702585:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3491 00000100:00080000:3.0:1775766869.702589:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766869.702594:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766869.702599:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766869.702611:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766869/real 1775766869] req@ffff8f9970eded80 x1862022967265408/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766869.702614:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967265792/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766869.702646:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967265408/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766869.702655:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766869.702658:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766869.702661:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967265408:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766869.702683:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967265536:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775766869.702709:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766869.702724:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766869/real 1775766869] req@ffff8f9970edd500 x1862022967265536/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766869.702765:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967265536/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766869.702774:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766869.702779:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766869.702782:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967265536:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775766869.702817:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967265664:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766869.702838:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766869.702848:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766869/real 1775766869] req@ffff8f9970ede680 x1862022967265664/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766869.702859:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967265664/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766869.702866:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766869.702869:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766869.702871:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967265664:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766869.702876:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967265792:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766869.702901:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766869.702911:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766869/real 1775766869] req@ffff8f9970edc000 x1862022967265792/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766869.702919:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967265792/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766869.702924:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766869.702926:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766869.702928:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967265792:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766870.597857:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766870.597905:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766871.621813:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766871.621885:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766872.645844:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766872.645914:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766873.669818:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766873.669862:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766874.693839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766874.693865:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766874.821813:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766874.821822:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766874.821827:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766874.821838:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3496 00000100:00080000:3.0:1775766874.821850:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766874.821856:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766874.821895:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967265920/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766874.821921:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766874.821924:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766874.821926:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766874.821929:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3496 00000100:00080000:3.0:1775766874.821932:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766874.821935:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766874.821942:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766874.821951:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967266048/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766874.821963:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766874.821965:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766874.821966:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766874.821969:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3496 00000100:00080000:3.0:1775766874.821971:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766874.821973:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766874.821975:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766874.821981:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967266176/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766874.821988:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766874.821990:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766874.821992:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766874.821994:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3496 00000100:00080000:3.0:1775766874.821996:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766874.821998:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775766874.822001:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766874.822007:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967266304/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766874.822014:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967265920:192.168.202.146@tcp:250:kworker.0 00000800:00000100:0.0:1775766874.822072:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766874.822115:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766874/real 1775766874] req@ffff8f9970edf800 x1862022967265920/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766874.822130:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967265920/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766874.822138:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766874.822142:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766874.822145:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967265920:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766874.822150:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967266048:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775766874.822164:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766874.822172:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766874/real 1775766874] req@ffff8f9970edf480 x1862022967266048/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766874.822179:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967266048/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766874.822184:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766874.822186:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766874.822188:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967266048:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775766874.822192:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967266176:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766874.822202:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766874.822207:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766874/real 1775766874] req@ffff8f9970edd880 x1862022967266176/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766874.822214:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967266176/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766874.822273:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766874.822276:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766874.822279:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967266176:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766874.822301:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967266304:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766874.822316:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766874.822323:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766874/real 1775766874] req@ffff8f9970ede300 x1862022967266304/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766874.822330:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967266304/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766874.822336:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766874.822338:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766874.822340:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967266304:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766875.717876:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766875.717885:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766876.741845:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766876.741933:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766877.765821:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766877.765872:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766878.789890:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766878.789963:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766879.813877:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766879.813878:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766879.941892:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766879.941906:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766879.941913:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766879.941926:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3501 00000100:00080000:0.0:1775766879.941941:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766879.941949:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766879.942002:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967266432/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766879.942039:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775766879.942068:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967266432:192.168.202.146@tcp:250:kworker.0 00000800:00000100:3.0:1775766879.942145:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775766879.942179:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766879.942185:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775766879.942194:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766879/real 1775766879] req@ffff8f9970a77480 x1862022967266432/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766879.942198:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3501 00000100:00080000:0.0:1775766879.942227:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775766879.942234:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967266432/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766879.942239:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00080000:3.0:1775766879.942260:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766879.942266:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000008:00100000:0.0:1775766879.942268:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766879.942270:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967266432:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766879.942311:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967266560/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766879.942340:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766879.942342:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766879.942344:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766879.942347:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3501 00000100:00080000:0.0:1775766879.942351:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766879.942355:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766879.942361:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766879.942365:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967266560:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775766879.942381:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967266688/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766879.942392:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1775766879.942398:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775766879.942404:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766879.942407:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766879.942414:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3501 00000100:00080000:0.0:1775766879.942418:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766879.942423:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766879.942423:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766879/real 1775766879] req@ffff8f9970a75c00 x1862022967266560/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766879.942442:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967266560/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766879.942449:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766879.942457:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000008:00100000:0.0:1775766879.942458:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766879.942465:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967266560:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775766879.942477:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967266816/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766879.942484:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967266688:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766879.942513:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766879.942522:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766879/real 1775766879] req@ffff8f9970a76a00 x1862022967266688/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766879.942531:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967266688/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766879.942537:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766879.942541:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766879.942543:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967266688:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766879.942551:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967266816:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766879.942561:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766879.942567:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766879/real 1775766879] req@ffff8f9970a77800 x1862022967266816/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766879.942574:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967266816/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766879.942584:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766879.942586:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766879.942588:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967266816:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766880.837875:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766880.837968:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766881.861834:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766881.861897:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766882.885912:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766882.885942:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775766883.909840:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766883.909901:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766884.933849:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766884.933853:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766885.061962:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766885.061975:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766885.061982:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766885.061997:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3506 00000100:00080000:0.0:1775766885.062015:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766885.062026:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766885.062079:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967266944/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766885.062115:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766885.062119:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766885.062123:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766885.062133:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3506 00000100:00080000:0.0:1775766885.062137:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766885.062142:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766885.062153:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766885.062171:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967266944:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766885.062186:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967267072/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766885.062202:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766885.062206:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766885.062208:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766885.062214:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3506 00000800:00000100:3.0:1775766885.062241:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775766885.062281:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766885.062289:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766885.062290:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766885/real 1775766885] req@ffff8f9970a76d80 x1862022967266944/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1775766885.062305:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766885.062316:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967266944/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766885.062326:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967267200/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766885.062327:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766885.062339:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766885.062352:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967266944:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775766885.062369:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766885.062374:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766885.062376:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775766885.062378:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967267072:192.168.202.146@tcp:38:kworker.0 00000100:00080000:0.0:1775766885.062388:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3506 00000100:00080000:0.0:1775766885.062396:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766885.062401:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:3.0:1775766885.062406:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:0.0:1775766885.062408:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766885.062417:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766885/real 1775766885] req@ffff8f9970a74380 x1862022967267072/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766885.062430:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967267328/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766885.062431:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967267072/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766885.062478:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766885.062482:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766885.062489:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967267072:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766885.062502:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967267200:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766885.062516:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766885.062522:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766885/real 1775766885] req@ffff8f9970a74700 x1862022967267200/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766885.062530:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967267200/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766885.062536:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766885.062540:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766885.062543:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967267200:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766885.062559:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967267328:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766885.062580:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766885.062592:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766885/real 1775766885] req@ffff8f9970a76d80 x1862022967267328/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766885.062603:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967267328/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766885.062612:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766885.062615:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766885.062618:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967267328:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766885.957885:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766885.957929:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766886.981836:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766886.981908:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766888.005821:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766888.005869:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766889.029809:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766889.029879:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766890.053824:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766890.053862:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00080000:0.0:1775766890.181889:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766890.181902:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766890.181908:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766890.181923:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3511 00000100:00080000:0.0:1775766890.181938:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766890.181947:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766890.182000:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967267456/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766890.182038:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766890.182080:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766890.182082:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766890.182087:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3511 00000100:00080000:0.0:1775766890.182091:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766890.182095:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766890.182120:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766890.182137:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967267584/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766890.182152:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766890.182158:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766890.182160:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775766890.182160:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967267456:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775766890.182174:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3511 00000100:00080000:0.0:1775766890.182179:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766890.182183:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766890.182189:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766890.182200:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967267712/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766890.182209:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766890.182213:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766890.182215:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766890.182218:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3511 00000100:00080000:0.0:1775766890.182221:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766890.182224:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766890.182240:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766890.182246:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766890.182259:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967267840/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766890.182290:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766890/real 1775766890] req@ffff8f9970a77800 x1862022967267456/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766890.182306:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967267456/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766890.182315:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766890.182318:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766890.182321:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967267456:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766890.182340:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967267584:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766890.182355:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766890.182363:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766890/real 1775766890] req@ffff8f9970a76a00 x1862022967267584/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766890.182371:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967267584/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766890.182376:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766890.182378:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766890.182380:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967267584:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766890.182384:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967267712:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766890.182394:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766890.182400:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766890/real 1775766890] req@ffff8f9970a75c00 x1862022967267712/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766890.182421:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967267712/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766890.182427:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766890.182429:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766890.182431:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967267712:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766890.182435:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967267840:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766890.182447:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766890.182453:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766890/real 1775766890] req@ffff8f9970a77480 x1862022967267840/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766890.182460:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967267840/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766890.182466:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766890.182468:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766890.182469:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967267840:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766891.077810:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766891.077889:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766892.101923:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766892.101978:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766893.125846:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766893.125907:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766894.149863:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766894.149965:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766895.173839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766895.173841:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766895.301854:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766895.301862:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766895.301866:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766895.301875:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3516 00000100:00080000:1.0:1775766895.301885:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766895.301892:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766895.301926:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967267968/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766895.301954:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766895.301958:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766895.301961:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766895.301965:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3516 00000100:00080000:1.0:1775766895.301969:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766895.301985:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766895.301991:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967267968:192.168.202.146@tcp:250:kworker.0 00000008:00100000:1.0:1775766895.301996:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766895.302014:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967268096/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766895.302026:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766895.302028:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766895.302030:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1775766895.302078:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775766895.302085:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3516 00000100:00080000:1.0:1775766895.302094:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766895.302098:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766895.302103:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766895.302114:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967268224/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766895.302122:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766895.302125:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1775766895.302125:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766895/real 1775766895] req@ffff8f994f2e6680 x1862022967267968/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766895.302128:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766895.302136:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3516 00000100:00080000:1.0:1775766895.302141:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766895.302144:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766895.302148:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766895.302152:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967267968/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766895.302160:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967268352/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766895.302163:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766895.302171:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766895.302177:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967267968:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766895.302197:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967268096:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766895.302215:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766895.302224:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766895/real 1775766895] req@ffff8f994f2e4000 x1862022967268096/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766895.302232:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967268096/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766895.302238:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766895.302240:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766895.302242:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967268096:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766895.302246:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967268224:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766895.302256:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766895.302262:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766895/real 1775766895] req@ffff8f994f2e5f80 x1862022967268224/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766895.302269:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967268224/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766895.302274:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766895.302276:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766895.302277:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967268224:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766895.302282:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967268352:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766895.302291:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766895.302296:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766895/real 1775766895] req@ffff8f994f2e6a00 x1862022967268352/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766895.302303:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967268352/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766895.302308:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766895.302310:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766895.302312:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967268352:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766896.197883:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766896.197983:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766897.221985:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766897.222003:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775766898.245901:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766898.245938:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766899.269842:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766899.269940:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766900.293847:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766900.293890:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766900.421879:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766900.421891:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766900.421895:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766900.421907:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3522 00000100:00080000:1.0:1775766900.421920:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766900.421928:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766900.422045:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967268480/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766900.422081:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766900.422085:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766900.422087:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766900.422092:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3522 00000100:00080000:1.0:1775766900.422098:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766900.422104:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766900.422116:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766900.422133:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967268608/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766900.422136:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967268480:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775766900.422168:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766900.422172:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766900.422173:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766900.422191:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3522 00000100:00080000:1.0:1775766900.422194:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766900.422197:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766900.422202:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766900.422221:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967268736/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766900.422231:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766900.422235:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766900.422237:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766900.422288:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3522 00000100:00080000:1.0:1775766900.422296:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766900.422300:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766900.422308:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766900.422329:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766900.422351:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967268864/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766900.422451:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766900/real 1775766900] req@ffff8f994f2e4700 x1862022967268480/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766900.422474:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967268480/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766900.422486:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766900.422496:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766900.422501:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967268480:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766900.422533:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967268608:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766900.422557:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766900.422573:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766900/real 1775766900] req@ffff8f994f2e6300 x1862022967268608/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766900.422585:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967268608/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766900.422592:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766900.422596:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766900.422598:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967268608:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766900.422603:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967268736:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766900.422617:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766900.422625:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766900/real 1775766900] req@ffff8f994f2e4e00 x1862022967268736/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766900.422633:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967268736/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766900.422639:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766900.422641:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766900.422643:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967268736:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766900.422648:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967268864:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766900.422660:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766900.422676:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766900/real 1775766900] req@ffff8f994f2e7b80 x1862022967268864/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766900.422685:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967268864/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766900.422691:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766900.422694:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766900.422696:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967268864:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766901.317821:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766901.318067:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766902.341827:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766902.341925:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766903.365826:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766903.365994:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766904.390014:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766904.390044:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766905.413818:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766905.413835:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00080000:1.0:1775766905.541988:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766905.541997:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766905.542001:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766905.542012:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3527 00000100:00080000:1.0:1775766905.542023:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766905.542032:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766905.542066:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967268992/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766905.542090:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766905.542092:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766905.542094:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766905.542097:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3527 00000100:00080000:1.0:1775766905.542099:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766905.542102:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766905.542117:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766905.542129:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967268992:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766905.542165:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967269120/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766905.542175:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766905.542177:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766905.542178:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766905.542181:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3527 00000100:00080000:1.0:1775766905.542184:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775766905.542188:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775766905.542190:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766905.542196:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766905.542203:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967269248/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766905.542210:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766905.542213:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766905.542214:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766905.542216:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3527 00000100:00080000:1.0:1775766905.542218:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766905.542220:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766905.542224:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766905.542231:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967269376/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766905.542236:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766905/real 1775766905] req@ffff8f994f2e7b80 x1862022967268992/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766905.542255:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967268992/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766905.542263:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766905.542267:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766905.542270:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967268992:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766905.542288:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967269120:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775766905.542304:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766905.542313:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766905/real 1775766905] req@ffff8f994f2e4e00 x1862022967269120/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766905.542321:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967269120/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766905.542328:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766905.542331:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766905.542333:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967269120:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775766905.542469:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967269248:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766905.542549:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766905.542592:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766905/real 1775766905] req@ffff8f994f2e6300 x1862022967269248/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766905.542627:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967269248/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766905.542646:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766905.542655:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766905.542662:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967269248:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766905.542671:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967269376:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775766905.542696:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766905.542708:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766905/real 1775766905] req@ffff8f994f2e4700 x1862022967269376/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766905.542722:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967269376/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766905.542732:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766905.542736:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766905.542740:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967269376:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766906.437871:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766906.437924:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766907.461899:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766907.461915:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766908.485839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766908.486048:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766909.509818:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766909.509950:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766910.533830:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766910.533835:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766910.662015:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766910.662102:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766910.662109:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766910.662124:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3532 00000100:00080000:0.0:1775766910.662142:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766910.662152:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766910.662214:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967269504/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766910.662256:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766910.662262:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766910.662266:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766910.662272:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3532 00000100:00080000:0.0:1775766910.662280:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766910.662285:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766910.662300:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766910.662304:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967269504:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766910.662319:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967269632/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766910.662337:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766910.662343:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766910.662346:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766910.662352:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3532 00000100:00080000:0.0:1775766910.662357:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766910.662362:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766910.662368:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766910.662379:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967269760/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766910.662415:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766910.662420:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766910.662423:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766910.662431:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3532 00000100:00080000:0.0:1775766910.662437:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766910.662442:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766910.662452:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766910.662456:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967269632:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775766910.662473:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967269888/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766910.662510:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967269760:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766910.662553:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967269888:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766911.557969:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766911.558001:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766912.581860:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766912.581940:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766913.605839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766913.605941:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775766913.734072:0:2366:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:1.0:1775766913.734166:0:2366:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775766913.734183:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775766913.734256:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775766913.734269:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775766913.734282:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:3.0:1775766913.734307:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766910/real 1775766913] req@ffff8f9970a77100 x1862022967269888/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766913.734357:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967269888/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766913.734384:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766913.734391:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766913.734397:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967269888:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766913.734410:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766910/real 1775766913] req@ffff8f9970a75180 x1862022967269632/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766913.734427:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967269632/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766913.734445:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766913.734450:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766913.734454:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967269632:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766913.734463:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766910/real 1775766913] req@ffff8f9970a76300 x1862022967269760/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766913.734490:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967269760/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766913.734501:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766913.734505:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766913.734510:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967269760:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766913.734518:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766910/real 1775766913] req@ffff8f9970a75500 x1862022967269504/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766913.734550:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967269504/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766913.734561:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766913.734566:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766913.734570:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967269504:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766914.629889:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766914.629912:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766915.653988:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766915.654032:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766915.781933:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766915.781944:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766915.781949:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766915.781959:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3537 00000100:00080000:1.0:1775766915.781971:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766915.781978:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766915.782017:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967270016/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766915.782048:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766915.782052:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766915.782054:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766915.782057:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3537 00000100:00080000:1.0:1775766915.782059:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766915.782062:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766915.782072:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766915.782084:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967270144/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766915.782096:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775766915.782120:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967270016:192.168.202.146@tcp:250:kworker.0 00000800:00000100:3.0:1775766915.782215:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775766915.782227:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766915.782233:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766915.782250:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3537 00000100:00080000:1.0:1775766915.782264:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766915.782271:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766915.782287:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766915.782299:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766915/real 1775766915] req@ffff8f994f2e6a00 x1862022967270016/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766915.782337:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967270016/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766915.782343:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967270272/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766915.782378:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766915.782382:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766915.782389:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00080000:1.0:1775766915.782392:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766915.782398:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775766915.782404:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967270016:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775766915.782410:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3537 00000100:00100000:3.0:1775766915.782425:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967270144:192.168.202.146@tcp:38:kworker.0 00000100:00080000:1.0:1775766915.782427:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766915.782449:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766915.782457:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766915.782480:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766915.782483:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967270400/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766915.782504:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766915/real 1775766915] req@ffff8f994f2e5f80 x1862022967270144/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766915.782523:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967270144/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766915.782535:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766915.782543:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766915.782548:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967270144:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766915.782597:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967270272:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766915.782624:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766915.782637:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766915/real 1775766915] req@ffff8f994f2e4000 x1862022967270272/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766915.782650:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967270272/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766915.782666:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766915.782670:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766915.782673:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967270272:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766915.782681:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967270400:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766915.782703:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766915.782713:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766915/real 1775766915] req@ffff8f994f2e6680 x1862022967270400/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766915.782725:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967270400/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766915.782733:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766915.782776:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766915.782780:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967270400:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766916.677897:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766916.677928:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766917.701830:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766917.701930:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766918.725953:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766918.726055:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766919.749881:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766919.749950:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766920.773854:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766920.773883:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766920.901869:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766920.901920:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766920.901924:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766920.901935:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3542 00000100:00080000:2.0:1775766920.901946:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766920.901953:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766920.901992:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2d80 x1862022967270528/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766920.902021:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766920.902026:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766920.902027:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766920.902032:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3542 00000100:00080000:2.0:1775766920.902035:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766920.902039:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766920.902048:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766920.902063:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0e00 x1862022967270656/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766920.902075:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775766920.902076:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967270528:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775766920.902083:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766920.902085:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766920.902090:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3542 00000100:00080000:2.0:1775766920.902094:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766920.902098:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766920.902103:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766920.902116:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0700 x1862022967270784/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766920.902139:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775766920.902150:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766920.902154:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766920.902156:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766920.902161:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3542 00000100:00080000:2.0:1775766920.902166:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766920.902171:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766920.902176:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766920.902189:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1500 x1862022967270912/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766920.902195:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766920/real 1775766920] req@ffff8f99703d2d80 x1862022967270528/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766920.902223:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2d80 x1862022967270528/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766920.902235:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766920.902239:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766920.902244:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967270528:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766920.902266:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967270656:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766920.902286:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766920.902299:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766920/real 1775766920] req@ffff8f99703d0e00 x1862022967270656/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766920.902313:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0e00 x1862022967270656/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766920.902322:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766920.902325:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766920.902329:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967270656:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766920.902336:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967270784:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766920.902357:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766920.902368:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766920/real 1775766920] req@ffff8f99703d0700 x1862022967270784/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766920.902379:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0700 x1862022967270784/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766920.902386:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766920.902389:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766920.902393:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967270784:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766920.902399:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967270912:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766920.902418:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766920.902439:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766920/real 1775766920] req@ffff8f99703d1500 x1862022967270912/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766920.902448:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1500 x1862022967270912/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766920.902454:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766920.902456:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766920.902458:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967270912:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766921.797874:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766921.797918:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766922.821851:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766922.821949:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766923.845878:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766923.845947:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766924.869876:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766924.870032:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766925.893887:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766925.893995:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00080000:0.0:1775766926.021859:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766926.021868:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766926.021872:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766926.021881:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3547 00000100:00080000:0.0:1775766926.021890:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766926.021896:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766926.021929:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967271040/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766926.021953:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775766926.022018:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967271040:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775766926.022063:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766926.022066:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766926.022072:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3547 00000100:00080000:0.0:1775766926.022080:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766926.022086:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766926.022099:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766926.022123:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967271168/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766926.022137:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766926.022140:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766926.022142:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766926.022145:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3547 00000100:00080000:0.0:1775766926.022148:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1775766926.022152:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775766926.022154:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766926.022160:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766926.022168:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967271296/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766926.022175:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766926.022179:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766926.022181:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766926.022187:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3547 00000100:00080000:0.0:1775766926.022191:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766926.022193:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766926.022216:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766926/real 1775766926] req@ffff8f9970a75500 x1862022967271040/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1775766926.022221:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766926.022235:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967271424/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766926.022242:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967271040/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766926.022254:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766926.022258:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766926.022261:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967271040:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766926.022286:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967271168:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766926.022313:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766926.022323:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766926/real 1775766926] req@ffff8f9970a76300 x1862022967271168/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766926.022331:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967271168/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766926.022337:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766926.022340:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766926.022342:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967271168:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766926.022346:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967271296:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766926.022357:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766926.022363:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766926/real 1775766926] req@ffff8f9970a75180 x1862022967271296/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766926.022371:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967271296/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766926.022376:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766926.022378:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766926.022380:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967271296:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766926.022384:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967271424:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766926.022394:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766926.022400:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766926/real 1775766926] req@ffff8f9970a77100 x1862022967271424/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766926.022413:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967271424/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766926.022419:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766926.022421:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766926.022423:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967271424:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766926.917795:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766926.917870:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766927.941822:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766927.941883:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766928.965892:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766928.965971:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766929.989811:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766929.989868:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766931.013858:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766931.013889:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766931.141880:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766931.141889:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766931.141892:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766931.141901:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3552 00000100:00080000:2.0:1775766931.141911:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766931.141916:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766931.141988:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3100 x1862022967271552/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766931.142021:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766931.142025:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766931.142027:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766931.142033:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3552 00000100:00080000:2.0:1775766931.142037:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766931.142043:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766931.142054:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766931.142073:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2300 x1862022967271680/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766931.142075:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967271552:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775766931.142091:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766931.142094:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766931.142096:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766931.142099:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3552 00000100:00080000:2.0:1775766931.142103:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766931.142106:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766931.142111:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766931.142123:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1180 x1862022967271808/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766931.142131:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766931.142134:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766931.142135:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766931.142139:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3552 00000100:00080000:2.0:1775766931.142141:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1775766931.142146:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775766931.142147:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766931.142154:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766931.142164:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3480 x1862022967271936/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766931.142202:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766931/real 1775766931] req@ffff8f99703d3100 x1862022967271552/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766931.142221:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3100 x1862022967271552/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766931.142231:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766931.142237:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766931.142241:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967271552:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766931.142261:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967271680:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766931.142278:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766931.142287:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766931/real 1775766931] req@ffff8f99703d2300 x1862022967271680/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766931.142296:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2300 x1862022967271680/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766931.142303:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766931.142305:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766931.142307:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967271680:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766931.142312:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967271808:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766931.142326:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766931.142333:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766931/real 1775766931] req@ffff8f99703d1180 x1862022967271808/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766931.142341:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1180 x1862022967271808/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766931.142346:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766931.142348:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766931.142350:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967271808:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766931.142363:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967271936:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766931.142376:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766931.142383:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766931/real 1775766931] req@ffff8f99703d3480 x1862022967271936/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766931.142391:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3480 x1862022967271936/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766931.142396:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766931.142398:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766931.142400:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967271936:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766932.037847:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766932.037907:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766933.061876:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766933.061895:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766934.085815:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766934.085861:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766935.109837:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766935.109914:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766936.133820:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766936.133832:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766936.262505:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766936.262517:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766936.262522:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766936.262535:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3557 00000100:00080000:2.0:1775766936.262549:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766936.262557:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766936.262608:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1880 x1862022967272064/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766936.262662:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766936.262666:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766936.262668:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766936.262680:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3557 00000100:00080000:2.0:1775766936.262688:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766936.262692:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766936.262703:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766936.262703:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967272064:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766936.262723:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2a00 x1862022967272192/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766936.262735:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766936.262761:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766936.262763:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766936.262768:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3557 00000100:00080000:2.0:1775766936.262773:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766936.262777:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766936.262783:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766936.262799:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1c00 x1862022967272320/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766936.262811:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766936.262814:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766936.262816:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766936.262819:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3557 00000100:00080000:2.0:1775766936.262824:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1775766936.262827:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775766936.262832:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766936.262838:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766936.262850:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3b80 x1862022967272448/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766936.262884:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766936/real 1775766936] req@ffff8f99703d1880 x1862022967272064/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766936.262911:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1880 x1862022967272064/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766936.262922:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766936.262926:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766936.262930:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967272064:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766936.262953:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967272192:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766936.262972:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766936.262981:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766936/real 1775766936] req@ffff8f99703d2a00 x1862022967272192/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766936.262990:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2a00 x1862022967272192/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766936.262996:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766936.262999:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766936.263001:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967272192:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766936.263005:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967272320:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766936.263017:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766936.263023:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766936/real 1775766936] req@ffff8f99703d1c00 x1862022967272320/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766936.263030:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1c00 x1862022967272320/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766936.263040:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766936.263042:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766936.263044:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967272320:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766936.263047:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967272448:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766936.263057:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766936.263065:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766936/real 1775766936] req@ffff8f99703d3b80 x1862022967272448/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766936.263077:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3b80 x1862022967272448/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766936.263084:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766936.263088:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766936.263091:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967272448:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766937.157800:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766937.157863:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766938.181827:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766938.181901:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766939.205839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766939.205897:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766940.229874:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766940.229886:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766941.253848:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766941.253869:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766941.381842:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766941.381852:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766941.381857:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766941.381868:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3562 00000100:00080000:2.0:1775766941.381882:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766941.381890:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766941.381935:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3b80 x1862022967272576/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766941.381966:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766941.381972:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766941.381974:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766941.381981:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3562 00000100:00080000:2.0:1775766941.381986:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766941.382007:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766941.382017:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766941.382019:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967272576:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766941.382038:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1c00 x1862022967272704/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766941.382052:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766941.382055:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766941.382057:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766941.382060:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3562 00000100:00080000:2.0:1775766941.382072:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766941.382075:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:3.0:1775766941.382080:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:2.0:1775766941.382084:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766941.382097:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2a00 x1862022967272832/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766941.382104:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766941.382107:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766941.382109:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766941.382112:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3562 00000100:00080000:2.0:1775766941.382115:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766941.382117:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766941.382121:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766941.382128:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766941/real 1775766941] req@ffff8f99703d3b80 x1862022967272576/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766941.382135:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1880 x1862022967272960/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766941.382150:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3b80 x1862022967272576/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766941.382159:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766941.382163:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766941.382167:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967272576:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766941.382187:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967272704:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766941.382216:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766941.382225:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766941/real 1775766941] req@ffff8f99703d1c00 x1862022967272704/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766941.382254:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1c00 x1862022967272704/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766941.382262:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766941.382266:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766941.382271:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967272704:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766941.382277:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967272832:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766941.382301:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766941.382310:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766941/real 1775766941] req@ffff8f99703d2a00 x1862022967272832/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766941.382320:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2a00 x1862022967272832/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766941.382327:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766941.382330:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766941.382333:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967272832:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766941.382338:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967272960:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766941.382352:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766941.382364:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766941/real 1775766941] req@ffff8f99703d1880 x1862022967272960/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766941.382377:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1880 x1862022967272960/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766941.382384:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766941.382388:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766941.382391:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967272960:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766942.277883:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766942.277970:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766943.301857:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766943.301930:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766944.325846:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766944.325969:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766945.349833:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766945.349934:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766946.373817:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:3.0:1775766946.373857:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766946.501963:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766946.501981:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766946.501989:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766946.502006:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3568 00000100:00080000:0.0:1775766946.502026:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766946.502038:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766946.502110:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967273088/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766946.502160:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766946.502165:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766946.502167:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766946.502172:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3568 00000100:00080000:0.0:1775766946.502178:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766946.502183:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766946.502196:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766946.502213:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967273216/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766946.502225:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766946.502229:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766946.502231:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766946.502236:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3568 00000100:00080000:0.0:1775766946.502239:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766946.502243:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766946.502249:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766946.502255:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967273088:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775766946.502265:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967273344/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766946.502278:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766946.502282:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766946.502284:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766946.502289:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3568 00000100:00080000:0.0:1775766946.502292:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766946.502295:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766946.502301:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766946.502316:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967273472/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766946.502382:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766946.502474:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766946/real 1775766946] req@ffff8f9970a77480 x1862022967273088/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766946.502543:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967273088/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766946.502569:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766946.502574:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766946.502579:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967273088:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766946.502589:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967273216:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766946.502621:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766946.502648:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766946/real 1775766946] req@ffff8f9970a75c00 x1862022967273216/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766946.502663:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967273216/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766946.502673:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766946.502677:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766946.502680:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967273216:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766946.502735:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967273344:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766946.502805:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766946.502821:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766946/real 1775766946] req@ffff8f9970a76a00 x1862022967273344/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766946.502851:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967273344/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766946.502862:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766946.502866:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766946.502870:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967273344:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766946.502878:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967273472:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766946.502901:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766946.502913:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766946/real 1775766946] req@ffff8f9970a77800 x1862022967273472/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766946.502927:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967273472/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766946.502945:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766946.502949:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766946.502953:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967273472:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766947.397933:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766947.397939:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766948.421824:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766948.421911:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766949.445871:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766949.445946:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766950.469844:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766950.469936:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766951.493836:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775766951.493875:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766951.621939:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766951.621949:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766951.622021:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766951.622033:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3573 00000100:00080000:0.0:1775766951.622045:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766951.622053:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766951.622101:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967273600/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766951.622173:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967273600:192.168.202.146@tcp:250:kworker.0 00000800:00000100:3.0:1775766951.622231:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775766951.622261:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766951.622265:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766951.622268:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766951.622273:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3573 00000100:00080000:0.0:1775766951.622278:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766951.622282:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766951.622292:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766951.622308:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967273728/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766951.622421:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766951/real 1775766951] req@ffff8f9970a76d80 x1862022967273600/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766951.622443:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967273600/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766951.622455:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766951.622476:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766951.622481:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967273600:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766951.622505:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967273728:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766951.622531:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766951.622545:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766951/real 1775766951] req@ffff8f9970a74700 x1862022967273728/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766951.622559:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967273728/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766951.622567:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766951.622571:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766951.622574:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967273728:192.168.202.146@tcp:38:kworker.0 00000100:00080000:0.0:1775766951.622631:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766951.622640:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766951.622642:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766951.622647:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3573 00000100:00080000:0.0:1775766951.622653:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766951.622658:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766951.622666:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766951.622680:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967273856/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766951.622707:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766951.622710:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766951.622712:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766951.622716:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3573 00000100:00080000:0.0:1775766951.622721:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766951.622725:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766951.622731:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766951.622753:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967273856:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766951.622777:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967273984/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766951.622789:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766951.622808:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766951/real 1775766951] req@ffff8f9970a74380 x1862022967273856/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766951.622820:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967273856/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766951.622826:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766951.622829:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766951.622831:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967273856:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766951.622841:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967273984:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766951.622856:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766951.622863:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766951/real 1775766951] req@ffff8f9970a74700 x1862022967273984/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766951.622871:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967273984/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766951.622876:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766951.622878:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766951.622880:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967273984:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766952.517864:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766952.517979:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766953.541817:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766953.541968:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766954.565938:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766954.566017:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766955.589816:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766955.589894:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766956.613842:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766956.613906:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766956.741867:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766956.741893:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766956.741896:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766956.741905:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3578 00000100:00080000:2.0:1775766956.741915:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766956.741921:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766956.741951:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3480 x1862022967274112/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766956.741975:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766956.741977:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766956.741979:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766956.741982:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3578 00000100:00080000:2.0:1775766956.741985:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766956.741987:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766956.741994:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766956.742003:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1180 x1862022967274240/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766956.742010:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766956.742012:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766956.742014:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766956.742016:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3578 00000100:00080000:2.0:1775766956.742018:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766956.742020:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766956.742026:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967274112:192.168.202.146@tcp:250:kworker.0 00000008:00100000:2.0:1775766956.742060:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766956.742070:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2300 x1862022967274368/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766956.742078:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766956.742080:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766956.742082:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1775766956.742086:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775766956.742089:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3578 00000100:00080000:2.0:1775766956.742093:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766956.742096:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766956.742099:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766956.742107:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3100 x1862022967274496/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766956.742137:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766956/real 1775766956] req@ffff8f99703d3480 x1862022967274112/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766956.742153:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3480 x1862022967274112/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766956.742162:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766956.742166:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766956.742170:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967274112:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766956.742176:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967274240:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766956.742192:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766956.742200:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766956/real 1775766956] req@ffff8f99703d1180 x1862022967274240/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766956.742208:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1180 x1862022967274240/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766956.742214:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766956.742216:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766956.742218:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967274240:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766956.742236:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967274368:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766956.742247:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766956.742253:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766956/real 1775766956] req@ffff8f99703d2300 x1862022967274368/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766956.742260:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2300 x1862022967274368/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766956.742274:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766956.742276:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766956.742278:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967274368:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766956.742289:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967274496:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766956.742309:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766956.742320:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766956/real 1775766956] req@ffff8f99703d3100 x1862022967274496/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766956.742334:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3100 x1862022967274496/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766956.742343:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766956.742347:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766956.742349:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967274496:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766957.637867:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766957.637945:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766958.661889:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766958.662032:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766959.685814:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766959.685861:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766960.709854:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766960.709913:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766961.733926:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766961.733961:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766961.861919:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766961.861928:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766961.861933:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766961.861941:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3583 00000100:00080000:1.0:1775766961.861952:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766961.861957:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766961.861992:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967274624/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766961.862018:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766961.862021:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766961.862023:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766961.862026:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3583 00000100:00080000:1.0:1775766961.862029:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766961.862039:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766961.862048:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766961.862061:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967274624:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775766961.862084:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967274752/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766961.862093:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766961.862096:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766961.862098:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766961.862101:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3583 00000100:00080000:1.0:1775766961.862104:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766961.862107:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766961.862111:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766961.862118:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967274880/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766961.862125:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766961.862128:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766961.862129:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766961.862134:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3583 00000800:00000100:3.0:1775766961.862136:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775766961.862138:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766961.862141:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766961.862144:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766961.862153:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967275008/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766961.862188:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766961/real 1775766961] req@ffff8f994f2e6d80 x1862022967274624/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766961.862211:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967274624/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766961.862223:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766961.862239:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766961.862244:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967274624:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766961.862270:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967274752:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766961.862308:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766961.862321:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766961/real 1775766961] req@ffff8f994f2e4a80 x1862022967274752/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766961.862335:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967274752/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766961.862345:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766961.862348:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766961.862382:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967274752:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766961.862389:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967274880:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766961.862424:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766961.862437:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766961/real 1775766961] req@ffff8f994f2e7800 x1862022967274880/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766961.862450:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967274880/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766961.862460:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766961.862463:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766961.862466:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967274880:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766961.862473:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967275008:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766961.862494:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766961.862506:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766961/real 1775766961] req@ffff8f994f2e5180 x1862022967275008/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766961.862521:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967275008/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766961.862536:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766961.862540:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766961.862544:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967275008:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766962.757827:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766962.757919:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766963.781808:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766963.781944:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766964.805836:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766964.806034:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766965.829809:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766965.829884:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766966.853848:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766966.853910:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766966.981867:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766966.981875:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766966.981879:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766966.981887:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3588 00000100:00080000:2.0:1775766966.981929:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766966.981935:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766966.981969:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1500 x1862022967275136/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766966.981997:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766966.982001:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766966.982003:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766966.982008:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3588 00000100:00080000:2.0:1775766966.982014:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766966.982020:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766966.982030:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766966.982044:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967275136:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766966.982045:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0700 x1862022967275264/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766966.982066:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766966.982068:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766966.982070:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766966.982075:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3588 00000100:00080000:2.0:1775766966.982077:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766966.982080:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766966.982085:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766966.982092:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0e00 x1862022967275392/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766966.982099:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766966.982114:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766966.982115:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766966.982118:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3588 00000100:00080000:2.0:1775766966.982121:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766966.982124:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766966.982128:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766966.982142:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775766966.982143:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2d80 x1862022967275520/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766966.982211:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766966/real 1775766966] req@ffff8f99703d1500 x1862022967275136/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766966.982233:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1500 x1862022967275136/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766966.982243:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766966.982247:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766966.982250:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967275136:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766966.982276:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967275264:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766966.982293:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766966.982328:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766966/real 1775766966] req@ffff8f99703d0700 x1862022967275264/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766966.982337:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0700 x1862022967275264/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766966.982343:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766966.982346:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766966.982348:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967275264:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766966.982353:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967275392:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766966.982367:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766966.982374:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766966/real 1775766966] req@ffff8f99703d0e00 x1862022967275392/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766966.982381:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0e00 x1862022967275392/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766966.982386:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766966.982388:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766966.982390:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967275392:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766966.982394:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967275520:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766966.982404:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766966.982410:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766966/real 1775766966] req@ffff8f99703d2d80 x1862022967275520/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766966.982417:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2d80 x1862022967275520/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766966.982422:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766966.982424:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766966.982426:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967275520:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766967.877795:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766967.877896:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766968.901900:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766968.901940:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766969.925876:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766969.925977:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766970.949820:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766970.949905:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766971.973904:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766971.974017:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766972.101808:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766972.101818:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766972.101824:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766972.101836:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3593 00000100:00080000:2.0:1775766972.101849:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766972.101857:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766972.101901:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0380 x1862022967275648/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766972.101929:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766972.101933:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766972.101934:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766972.101938:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3593 00000100:00100000:3.0:1775766972.101994:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967275648:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775766972.102021:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766972.102026:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766972.102046:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766972.102060:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0000 x1862022967275776/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766972.102072:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766972.102083:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:3.0:1775766972.102083:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775766972.102090:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766972.102096:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3593 00000100:00080000:2.0:1775766972.102102:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766972.102107:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766972.102120:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766972.102136:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2680 x1862022967275904/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766972.102149:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766972.102154:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766972.102157:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775766972.102162:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766972/real 1775766972] req@ffff8f99703d0380 x1862022967275648/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766972.102165:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3593 00000100:00080000:2.0:1775766972.102188:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766972.102195:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766972.102201:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766972.102213:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3800 x1862022967276032/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766972.102218:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0380 x1862022967275648/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766972.102236:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766972.102264:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766972.102272:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967275648:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766972.102323:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967275776:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766972.102354:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766972.102369:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766972/real 1775766972] req@ffff8f99703d0000 x1862022967275776/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766972.102385:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0000 x1862022967275776/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766972.102394:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766972.102398:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766972.102402:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967275776:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766972.102409:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967275904:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766972.102431:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766972.102444:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766972/real 1775766972] req@ffff8f99703d2680 x1862022967275904/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766972.102457:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2680 x1862022967275904/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766972.102473:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766972.102477:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766972.102481:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967275904:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766972.102487:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967276032:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766972.102506:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766972.102518:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766972/real 1775766972] req@ffff8f99703d3800 x1862022967276032/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766972.102531:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3800 x1862022967276032/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766972.102540:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766972.102543:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766972.102547:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967276032:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766972.997804:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766972.997922:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766974.021888:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766974.021978:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766975.045834:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766975.045884:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766976.069923:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766976.069952:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775766977.093888:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766977.093953:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766977.221865:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766977.221876:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766977.221881:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766977.221893:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3598 00000100:00080000:2.0:1775766977.221907:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766977.221914:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775766977.221959:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3800 x1862022967276160/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766977.221987:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766977.221990:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766977.221992:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766977.221997:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3598 00000100:00080000:2.0:1775766977.221999:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766977.222003:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766977.222014:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766977.222023:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967276160:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766977.222027:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2680 x1862022967276288/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766977.222037:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766977.222040:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766977.222064:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766977.222068:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3598 00000100:00080000:2.0:1775766977.222070:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766977.222074:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766977.222080:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766977.222091:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0000 x1862022967276416/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766977.222099:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766977.222102:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766977.222103:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766977.222107:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3598 00000100:00080000:2.0:1775766977.222111:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766977.222114:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775766977.222119:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766977.222131:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0380 x1862022967276544/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766977.222161:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967276288:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775766977.222220:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967276416:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766977.222249:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967276544:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766978.117833:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766978.117959:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766979.141834:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766979.141914:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766980.165873:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766980.166098:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775766980.294243:0:2368:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:1.0:1775766980.294309:0:2368:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775766980.294338:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775766980.294402:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775766980.294417:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775766980.294433:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:0.0:1775766980.294502:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766977/real 1775766980] req@ffff8f99703d2680 x1862022967276288/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766980.294566:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2680 x1862022967276288/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766980.294584:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766980.294592:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766980.294598:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967276288:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775766980.294611:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766977/real 1775766980] req@ffff8f99703d0000 x1862022967276416/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766980.294631:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0000 x1862022967276416/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766980.294644:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766980.294650:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766980.294655:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967276416:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766980.294667:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766977/real 1775766980] req@ffff8f99703d0380 x1862022967276544/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766980.294684:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0380 x1862022967276544/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766980.294703:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766980.294807:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766980.294811:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967276544:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775766980.294822:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766977/real 1775766980] req@ffff8f99703d3800 x1862022967276160/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766980.294845:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3800 x1862022967276160/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766980.294858:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766980.294863:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775766980.294868:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967276160:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775766981.189845:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766981.189918:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766982.213841:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766982.213968:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766982.341895:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766982.341904:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766982.341909:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766982.341917:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3603 00000100:00080000:0.0:1775766982.341944:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766982.341951:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766982.342003:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967276672/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766982.342029:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766982.342031:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766982.342033:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766982.342046:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3603 00000100:00080000:0.0:1775766982.342049:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766982.342051:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766982.342098:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967276672:192.168.202.146@tcp:250:kworker.0 00000008:00100000:0.0:1775766982.342115:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766982.342129:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967276800/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766982.342155:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766982.342158:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766982.342159:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766982.342162:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3603 00000100:00080000:0.0:1775766982.342166:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766982.342169:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766982.342174:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766982.342196:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766982.342198:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967276928/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766982.342207:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766982.342210:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766982.342212:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766982.342215:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3603 00000100:00080000:0.0:1775766982.342217:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766982.342220:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766982.342223:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766982.342231:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967277056/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766982.342239:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766982/real 1775766982] req@ffff8f9970a76d80 x1862022967276672/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766982.342260:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967276672/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766982.342286:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766982.342308:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766982.342312:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967276672:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766982.342338:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967276800:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766982.342361:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766982.342373:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766982/real 1775766982] req@ffff8f9970a77800 x1862022967276800/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766982.342385:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967276800/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766982.342393:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766982.342396:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766982.342400:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967276800:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766982.342405:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967276928:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766982.342423:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766982.342432:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766982/real 1775766982] req@ffff8f9970a76a00 x1862022967276928/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766982.342443:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967276928/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766982.342450:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766982.342453:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766982.342457:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967276928:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766982.342462:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967277056:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766982.342478:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766982.342488:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766982/real 1775766982] req@ffff8f9970a75c00 x1862022967277056/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766982.342500:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967277056/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766982.342508:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766982.342511:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766982.342514:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967277056:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766983.237913:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766983.237918:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766984.261823:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766984.261917:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766985.285875:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766985.285928:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766986.309802:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766986.309867:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766987.333839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766987.333892:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766987.461936:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766987.461949:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766987.461955:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766987.461970:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3609 00000100:00080000:0.0:1775766987.461987:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766987.461997:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775766987.462054:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967277184/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766987.462121:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766987.462126:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766987.462128:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766987.462134:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3609 00000100:00080000:0.0:1775766987.462138:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766987.462143:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766987.462157:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766987.462174:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967277312/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766987.462181:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967277184:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775766987.462193:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766987.462197:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766987.462199:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766987.462204:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3609 00000100:00080000:0.0:1775766987.462207:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766987.462210:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775766987.462217:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766987.462303:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775766987.462394:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967277440/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766987.462412:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775766987.462412:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766987/real 1775766987] req@ffff8f9970a77480 x1862022967277184/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766987.462426:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766987.462432:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766987.462447:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3609 00000100:00080000:0.0:1775766987.462462:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766987.462472:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766987.462483:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967277184/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1775766987.462484:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766987.462509:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967277568/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766987.462511:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766987.462531:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766987.462551:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967277184:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775766987.462597:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967277312:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775766987.462634:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766987.462654:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766987/real 1775766987] req@ffff8f9970a77100 x1862022967277312/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766987.462673:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967277312/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766987.462695:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766987.462703:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766987.462708:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967277312:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766987.462717:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967277440:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766987.462786:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766987.462804:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766987/real 1775766987] req@ffff8f9970a75180 x1862022967277440/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766987.462822:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967277440/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766987.462835:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766987.462840:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766987.462845:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967277440:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766987.462855:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967277568:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766987.462884:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766987.462899:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766987/real 1775766987] req@ffff8f9970a76300 x1862022967277568/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766987.462916:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967277568/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766987.462928:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766987.462932:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766987.462938:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967277568:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766988.357816:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766988.357897:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766989.381864:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766989.381912:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766990.405904:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766990.405911:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766991.429880:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766991.429976:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766992.453823:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766992.453892:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766992.581930:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766992.581942:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766992.581947:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766992.581968:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3614 00000100:00080000:1.0:1775766992.581997:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766992.582004:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766992.582105:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967277696/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766992.582131:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775766992.582168:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967277696:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775766992.582234:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766992.582239:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766992.582248:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3614 00000800:00000100:3.0:1775766992.582256:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775766992.582264:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766992.582272:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766992.582284:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766992.582317:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967277824/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766992.582329:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766992/real 1775766992] req@ffff8f994f2e5180 x1862022967277696/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766992.582334:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766992.582363:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766992.582366:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766992.582373:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3614 00000100:00080000:1.0:1775766992.582379:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775766992.582386:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967277696/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766992.582387:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766992.582400:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1775766992.582407:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766992.582414:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766992.582418:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967277952/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766992.582427:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967277696:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775766992.582449:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766992.582452:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766992.582454:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766992.582459:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3614 00000100:00080000:1.0:1775766992.582461:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766992.582464:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766992.582470:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967277824:192.168.202.146@tcp:38:kworker.0 00000008:00100000:1.0:1775766992.582486:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766992.582496:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775766992.582500:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967278080/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766992.582514:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766992/real 1775766992] req@ffff8f994f2e7800 x1862022967277824/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766992.582526:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967277824/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766992.582531:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766992.582534:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766992.582537:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967277824:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766992.582542:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967277952:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766992.582559:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766992.582567:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766992/real 1775766992] req@ffff8f994f2e4a80 x1862022967277952/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766992.582575:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967277952/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766992.582580:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766992.582582:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766992.582583:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967277952:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766992.582593:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967278080:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766992.582603:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766992.582610:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766992/real 1775766992] req@ffff8f994f2e6d80 x1862022967278080/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766992.582617:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967278080/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766992.582623:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766992.582625:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766992.582627:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967278080:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766993.477833:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766993.477942:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766994.501818:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766994.501936:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766995.525916:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766995.526036:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766996.549835:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766996.550045:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766997.573954:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775766997.573962:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766997.701860:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766997.701876:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766997.701883:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766997.701897:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3619 00000100:00080000:1.0:1775766997.701914:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766997.701924:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775766997.701991:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967278208/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766997.702058:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766997.702062:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766997.702064:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766997.702068:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3619 00000100:00080000:1.0:1775766997.702072:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766997.702085:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766997.702097:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766997.702128:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967278336/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766997.702137:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775766997.702141:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967278208:192.168.202.146@tcp:250:kworker.0 00000800:00000100:3.0:1775766997.702252:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775766997.702281:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766997.702297:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766997.702322:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3619 00000100:00080000:1.0:1775766997.702334:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775766997.702340:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766997/real 1775766997] req@ffff8f994f2e6680 x1862022967278208/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766997.702341:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775766997.702368:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766997.702380:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967278208/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766997.702392:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766997.702398:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775766997.702403:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967278464/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766997.702404:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967278208:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775766997.702428:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766997.702431:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766997.702432:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766997.702436:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3619 00000100:00080000:1.0:1775766997.702441:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766997.702445:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775766997.702445:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967278336:192.168.202.146@tcp:38:kworker.0 00000008:00100000:1.0:1775766997.702457:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766997.702468:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967278592/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766997.702482:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766997.702496:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766997/real 1775766997] req@ffff8f994f2e4000 x1862022967278336/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766997.702505:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967278336/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766997.702512:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766997.702515:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766997.702518:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967278336:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775766997.702527:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967278464:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766997.702540:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766997.702546:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766997/real 1775766997] req@ffff8f994f2e5f80 x1862022967278464/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766997.702553:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967278464/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766997.702559:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766997.702561:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766997.702563:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967278464:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775766997.702575:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967278592:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775766997.702586:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775766997.702592:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766997/real 1775766997] req@ffff8f994f2e6a00 x1862022967278592/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766997.702599:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967278592/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766997.702605:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766997.702607:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775766997.702608:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967278592:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775766998.597827:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766998.597927:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775766999.621876:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775766999.621932:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767000.645829:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767000.645905:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767001.669880:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767001.670003:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767002.693840:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767002.693971:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767002.822035:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767002.822065:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767002.822080:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767002.822099:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3624 00000100:00080000:0.0:1775767002.822119:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767002.822131:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767002.822215:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967278720/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767002.822417:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767002.822421:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767002.822425:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767002.822434:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3624 00000100:00080000:0.0:1775767002.822444:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767002.822452:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767002.822470:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767002.822510:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967278848/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767002.822520:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767002.822523:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767002.822524:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767002.822527:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3624 00000100:00080000:0.0:1775767002.822530:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767002.822534:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767002.822538:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767002.822548:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967278720:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767002.822549:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967278976/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767002.822561:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767002.822564:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767002.822565:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767002.822567:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3624 00000100:00080000:0.0:1775767002.822570:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767002.822573:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767002.822576:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767002.822585:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967279104/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775767002.822612:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767002.822657:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767002/real 1775767002] req@ffff8f9970a75500 x1862022967278720/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767002.822672:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967278720/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767002.822681:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767002.822685:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767002.822689:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967278720:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767002.822696:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967278848:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767002.822711:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767002.822720:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767002/real 1775767002] req@ffff8f9970a76300 x1862022967278848/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767002.822728:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967278848/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767002.822734:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767002.822757:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767002.822760:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967278848:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767002.822778:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967278976:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767002.822791:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767002.822798:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767002/real 1775767002] req@ffff8f9970a75180 x1862022967278976/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767002.822806:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967278976/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767002.822811:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767002.822813:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767002.822815:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967278976:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767002.822819:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967279104:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767002.822828:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767002.822834:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767002/real 1775767002] req@ffff8f9970a77100 x1862022967279104/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767002.822841:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967279104/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767002.822850:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767002.822853:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767002.822855:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967279104:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767003.717860:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767003.717954:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767004.741932:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767004.741946:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767005.765913:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767005.765933:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767006.789878:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767006.789987:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767007.813906:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767007.813920:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767007.941989:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767007.942002:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767007.942009:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767007.942024:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3629 00000100:00080000:0.0:1775767007.942044:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767007.942054:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767007.942120:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967279232/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767007.942163:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767007.942168:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767007.942170:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767007.942177:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3629 00000100:00080000:0.0:1775767007.942183:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767007.942189:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767007.942202:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767007.942209:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967279232:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767007.942224:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967279360/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767007.942246:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767007.942250:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767007.942252:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767007.942257:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3629 00000100:00080000:0.0:1775767007.942262:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767007.942265:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767007.942271:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767007.942293:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967279488/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775767007.942306:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775767007.942319:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767007.942324:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767007.942328:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767007.942335:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3629 00000100:00080000:0.0:1775767007.942340:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767007.942344:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767007.942350:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767007.942362:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967279616/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767007.942367:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767007/real 1775767007] req@ffff8f9970a77480 x1862022967279232/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767007.942446:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967279232/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767007.942456:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767007.942460:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767007.942464:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967279232:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767007.942490:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967279360:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767007.942518:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767007.942534:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767007/real 1775767007] req@ffff8f9970a75c00 x1862022967279360/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767007.942550:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967279360/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767007.942561:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767007.942567:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767007.942572:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967279360:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767007.942578:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967279488:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767007.942613:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767007.942621:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767007/real 1775767007] req@ffff8f9970a76a00 x1862022967279488/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767007.942628:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967279488/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767007.942634:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767007.942636:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767007.942638:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967279488:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767007.942643:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967279616:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767007.942664:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767007.942671:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767007/real 1775767007] req@ffff8f9970a77800 x1862022967279616/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767007.942679:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967279616/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767007.942684:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767007.942686:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767007.942688:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967279616:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767008.837940:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767008.838073:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767009.861840:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767009.861976:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767010.885952:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767010.886033:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767011.909982:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767011.909989:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767012.933917:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767012.934025:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767013.061891:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767013.061900:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767013.061906:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767013.061918:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3634 00000100:00080000:2.0:1775767013.061931:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767013.061938:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767013.062105:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2d80 x1862022967279744/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767013.062137:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767013.062141:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767013.062144:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767013.062151:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3634 00000100:00080000:2.0:1775767013.062163:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767013.062169:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767013.062177:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767013.062192:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0e00 x1862022967279872/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767013.062202:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767013.062204:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767013.062207:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767013.062210:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3634 00000100:00080000:2.0:1775767013.062213:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767013.062216:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767013.062221:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767013.062232:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0700 x1862022967280000/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767013.062237:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767013.062240:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767013.062241:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775767013.062256:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967279744:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775767013.062270:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3634 00000100:00080000:2.0:1775767013.062275:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767013.062280:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767013.062286:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767013.062301:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1500 x1862022967280128/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775767013.062441:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767013.062550:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767013/real 1775767013] req@ffff8f99703d2d80 x1862022967279744/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767013.062570:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2d80 x1862022967279744/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767013.062579:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767013.062583:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767013.062588:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967279744:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767013.062596:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967279872:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767013.062624:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767013.062634:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767013/real 1775767013] req@ffff8f99703d0e00 x1862022967279872/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767013.062642:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0e00 x1862022967279872/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767013.062648:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767013.062650:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767013.062653:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967279872:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767013.062657:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967280000:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767013.062669:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767013.062674:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767013/real 1775767013] req@ffff8f99703d0700 x1862022967280000/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767013.062682:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0700 x1862022967280000/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767013.062692:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767013.062694:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767013.062696:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967280000:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767013.062720:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967280128:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767013.062731:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767013.062757:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767013/real 1775767013] req@ffff8f99703d1500 x1862022967280128/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767013.062768:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1500 x1862022967280128/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767013.062776:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767013.062779:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767013.062782:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967280128:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767013.957921:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767013.958012:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767014.981880:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767014.981963:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767016.005913:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767016.005928:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767017.029842:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767017.029928:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767018.053857:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767018.053921:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767018.181872:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767018.181920:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767018.181923:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767018.181932:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3639 00000100:00080000:0.0:1775767018.181941:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767018.181947:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767018.181980:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967280256/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767018.182019:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767018.182022:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767018.182023:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767018.182027:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3639 00000100:00080000:0.0:1775767018.182031:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767018.182035:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767018.182045:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767018.182056:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967280256:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767018.182058:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967280384/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767018.182072:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767018.182074:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767018.182076:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767018.182078:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3639 00000100:00080000:0.0:1775767018.182081:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767018.182083:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767018.182088:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767018.182095:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967280512/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767018.182104:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767018.182107:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767018.182109:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767018.182112:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3639 00000100:00080000:0.0:1775767018.182114:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767018.182117:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767018.182120:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767018.182129:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967280640/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775767018.182138:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767018.182184:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767018/real 1775767018] req@ffff8f9970a76d80 x1862022967280256/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767018.182201:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967280256/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767018.182214:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767018.182219:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767018.182224:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967280256:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767018.182252:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967280384:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767018.182283:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767018.182296:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767018/real 1775767018] req@ffff8f9970a74700 x1862022967280384/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767018.182306:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967280384/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767018.182313:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767018.182316:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767018.182319:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967280384:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767018.182323:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967280512:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767018.182351:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767018.182359:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767018/real 1775767018] req@ffff8f9970a74380 x1862022967280512/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767018.182368:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967280512/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767018.182374:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767018.182377:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767018.182379:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967280512:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767018.182383:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967280640:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767018.182395:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767018.182401:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767018/real 1775767018] req@ffff8f9970a77800 x1862022967280640/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767018.182410:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967280640/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767018.182416:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767018.182418:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767018.182420:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967280640:192.168.202.146@tcp:8:kworker.0 00010000:00010000:0.0:1775767019.077907:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767019.078053:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775767020.101905:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767020.101977:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767021.125850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767021.125938:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767022.149854:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767022.149933:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767023.173834:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767023.173901:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767023.301892:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767023.301902:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767023.301907:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767023.301919:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3644 00000100:00080000:0.0:1775767023.301932:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767023.301938:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767023.301986:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967280768/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767023.302014:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767023.302017:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767023.302018:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767023.302022:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3644 00000100:00080000:0.0:1775767023.302026:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767023.302029:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767023.302039:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767023.302049:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967280896/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767023.302057:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767023.302060:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767023.302061:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767023.302064:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3644 00000100:00080000:0.0:1775767023.302066:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767023.302068:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767023.302072:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767023.302083:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967281024/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767023.302083:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967280768:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775767023.302108:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767023.302112:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767023.302113:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767023.302116:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3644 00000100:00080000:0.0:1775767023.302118:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767023.302121:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767023.302123:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767023.302137:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967281152/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775767023.302186:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767023.302242:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767023/real 1775767023] req@ffff8f9970a76a00 x1862022967280768/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767023.302263:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967280768/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767023.302276:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767023.302282:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767023.302287:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967280768:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767023.302297:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967280896:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767023.302325:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767023.302339:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767023/real 1775767023] req@ffff8f9970a75c00 x1862022967280896/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767023.302353:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967280896/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767023.302368:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767023.302372:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767023.302376:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967280896:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767023.302405:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967281024:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767023.302428:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767023.302440:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767023/real 1775767023] req@ffff8f9970a77480 x1862022967281024/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767023.302453:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967281024/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767023.302461:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767023.302464:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767023.302468:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967281024:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767023.302475:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967281152:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767023.302496:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767023.302508:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767023/real 1775767023] req@ffff8f9970a77100 x1862022967281152/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767023.302521:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967281152/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767023.302530:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767023.302534:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767023.302538:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967281152:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767024.197833:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767024.197918:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767025.221827:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767025.221879:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767026.245873:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767026.245926:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767027.269840:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767027.269904:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767028.293855:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767028.293908:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767028.421834:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767028.421841:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767028.421845:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767028.421853:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3650 00000100:00080000:0.0:1775767028.421864:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767028.421871:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767028.421908:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967281280/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767028.421936:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767028.421939:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767028.421940:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767028.421944:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3650 00000100:00080000:0.0:1775767028.421947:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767028.421950:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767028.421959:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767028.421973:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967281408/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767028.421987:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775767028.421987:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967281280:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775767028.421994:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767028.421998:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767028.422003:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3650 00000100:00080000:0.0:1775767028.422008:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767028.422012:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767028.422018:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767028.422032:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967281536/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767028.422043:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767028.422047:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767028.422050:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767028.422055:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3650 00000100:00080000:0.0:1775767028.422060:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1775767028.422060:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775767028.422067:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767028.422073:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767028.422088:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967281664/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767028.422107:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767028/real 1775767028] req@ffff8f9970a75180 x1862022967281280/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767028.422123:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967281280/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767028.422131:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767028.422134:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767028.422139:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967281280:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767028.422158:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967281408:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767028.422176:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767028.422186:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767028/real 1775767028] req@ffff8f9970a76300 x1862022967281408/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767028.422195:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967281408/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767028.422201:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767028.422204:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767028.422206:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967281408:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767028.422210:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967281536:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767028.422228:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767028.422235:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767028/real 1775767028] req@ffff8f9970a75500 x1862022967281536/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767028.422243:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967281536/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767028.422248:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767028.422250:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767028.422252:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967281536:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767028.422255:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967281664:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767028.422267:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767028.422277:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767028/real 1775767028] req@ffff8f9970a77100 x1862022967281664/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767028.422289:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967281664/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767028.422298:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767028.422301:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767028.422304:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967281664:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767029.317833:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767029.317870:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767030.341827:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767030.341909:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767031.365810:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767031.365864:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767032.389843:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767032.389897:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767033.413913:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767033.413978:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00080000:0.0:1775767033.541872:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767033.541901:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767033.541907:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767033.541928:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3655 00000100:00080000:0.0:1775767033.541943:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767033.541951:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767033.542006:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967281792/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767033.542046:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767033.542049:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767033.542051:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767033.542055:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3655 00000100:00080000:0.0:1775767033.542059:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767033.542062:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767033.542072:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767033.542086:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967281920/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767033.542095:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767033.542098:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767033.542099:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767033.542102:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3655 00000100:00080000:0.0:1775767033.542104:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767033.542107:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767033.542110:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767033.542117:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967281792:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767033.542118:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967282048/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767033.542132:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767033.542134:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767033.542135:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767033.542138:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3655 00000800:00000100:3.0:1775767033.542212:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775767033.542246:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767033.542257:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767033.542270:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767033.542279:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767033/real 1775767033] req@ffff8f9970a77480 x1862022967281792/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767033.542306:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967282176/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767033.542311:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967281792/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767033.542347:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767033.542355:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767033.542361:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967281792:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767033.542392:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967281920:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767033.542420:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767033.542434:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767033/real 1775767033] req@ffff8f9970a75c00 x1862022967281920/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767033.542447:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967281920/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767033.542455:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767033.542458:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767033.542461:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967281920:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767033.542468:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967282048:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767033.542489:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767033.542500:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767033/real 1775767033] req@ffff8f9970a76a00 x1862022967282048/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767033.542511:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967282048/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767033.542520:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767033.542523:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767033.542525:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967282048:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767033.542532:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967282176:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767033.542553:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767033.542564:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767033/real 1775767033] req@ffff8f9970a77800 x1862022967282176/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767033.542575:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967282176/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767033.542582:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767033.542586:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767033.542589:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967282176:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767034.437851:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767034.437994:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767035.461847:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767035.461917:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767036.485851:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767036.485922:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767037.509843:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767037.509870:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767038.533862:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767038.533922:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767038.661844:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767038.661853:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767038.661857:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767038.661871:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3660 00000100:00080000:2.0:1775767038.661883:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767038.661888:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767038.661924:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3100 x1862022967282304/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767038.661944:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767038.661946:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767038.661948:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767038.661951:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3660 00000100:00080000:2.0:1775767038.661953:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767038.661956:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767038.661964:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767038.661971:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2300 x1862022967282432/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767038.661979:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767038.661981:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767038.661983:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767038.661987:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3660 00000100:00080000:2.0:1775767038.661989:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767038.661991:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767038.662007:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967282304:192.168.202.146@tcp:250:kworker.0 00000008:00100000:2.0:1775767038.662024:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767038.662042:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1180 x1862022967282560/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767038.662054:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767038.662059:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767038.662061:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767038.662066:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3660 00000100:00080000:2.0:1775767038.662086:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767038.662090:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767038.662096:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775767038.662103:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767038.662107:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3480 x1862022967282688/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767038.662166:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767038/real 1775767038] req@ffff8f99703d3100 x1862022967282304/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767038.662192:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3100 x1862022967282304/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767038.662202:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767038.662205:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767038.662209:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967282304:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767038.662215:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967282432:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767038.662234:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767038.662243:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767038/real 1775767038] req@ffff8f99703d2300 x1862022967282432/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767038.662252:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2300 x1862022967282432/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767038.662258:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767038.662260:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767038.662263:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967282432:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767038.662288:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967282560:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767038.662300:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767038.662306:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767038/real 1775767038] req@ffff8f99703d1180 x1862022967282560/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767038.662314:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1180 x1862022967282560/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767038.662319:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767038.662321:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767038.662323:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967282560:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767038.662327:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967282688:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767038.662337:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767038.662342:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767038/real 1775767038] req@ffff8f99703d3480 x1862022967282688/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767038.662349:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3480 x1862022967282688/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767038.662360:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767038.662362:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767038.662364:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967282688:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767039.557808:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767039.557858:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767040.581876:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767040.581925:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767041.605832:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767041.605900:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767042.629839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767042.629905:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767043.653833:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767043.653937:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767043.781889:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767043.781900:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767043.781906:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767043.781916:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3665 00000100:00080000:0.0:1775767043.781929:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767043.781936:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767043.781981:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967282816/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767043.782007:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767043.782010:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767043.782011:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767043.782016:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3665 00000100:00080000:0.0:1775767043.782019:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767043.782022:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767043.782032:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767043.782040:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967282816:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767043.782044:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967282944/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767043.782053:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767043.782056:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767043.782057:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767043.782061:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3665 00000100:00080000:0.0:1775767043.782064:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767043.782066:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767043.782069:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767043.782078:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967283072/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767043.782085:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767043.782088:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767043.782089:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767043.782091:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3665 00000100:00080000:0.0:1775767043.782094:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767043.782096:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767043.782098:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767043.782106:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967283200/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767043.782180:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967282944:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767043.782214:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967283072:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767043.782242:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967283200:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767044.677832:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767044.677901:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767045.701833:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767045.701900:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767046.725879:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767046.725947:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775767046.854018:0:2367:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:1.0:1775767046.854068:0:2367:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775767046.854088:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775767046.854127:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775767046.854135:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775767046.854144:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:3.0:1775767046.854182:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767043/real 1775767046] req@ffff8f9970a74700 x1862022967282944/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767046.854219:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967282944/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767046.854232:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767046.854238:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767046.854242:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967282944:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767046.854253:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767043/real 1775767046] req@ffff8f9970a76d80 x1862022967283072/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767046.854267:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967283072/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767046.854275:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767046.854279:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767046.854283:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967283072:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767046.854292:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767043/real 1775767046] req@ffff8f9970a77800 x1862022967283200/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767046.854304:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967283200/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767046.854313:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767046.854317:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767046.854320:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967283200:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767046.854328:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767043/real 1775767046] req@ffff8f9970a74380 x1862022967282816/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767046.854349:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967282816/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767046.854357:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767046.854361:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767046.854364:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967282816:192.168.202.146@tcp:250:kworker.0 00010000:00010000:1.0:1775767047.749934:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767047.749947:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775767048.773845:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767048.773878:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767048.901825:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767048.901833:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767048.901868:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767048.901877:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3670 00000100:00080000:0.0:1775767048.901886:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767048.901892:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767048.901927:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967283328/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767048.901962:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767048.901965:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767048.901966:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767048.901969:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3670 00000100:00080000:0.0:1775767048.901972:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767048.901976:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767048.901985:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767048.901997:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967283456/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767048.902005:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767048.902009:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767048.902013:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767048.902022:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3670 00000100:00100000:2.0:1775767048.902023:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967283328:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775767048.902027:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767048.902044:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767048.902061:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767048.902080:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967283584/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767048.902096:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767048.902099:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767048.902101:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767048.902103:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3670 00000100:00080000:0.0:1775767048.902107:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767048.902110:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767048.902113:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767048.902121:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967283712/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767048.902142:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767048.902219:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767048/real 1775767048] req@ffff8f9970a76a00 x1862022967283328/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767048.902526:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967283328/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767048.902541:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767048.902547:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767048.902552:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967283328:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767048.902593:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967283456:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775767048.902998:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767048.903030:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767048/real 1775767048] req@ffff8f9970a75c00 x1862022967283456/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767048.903045:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967283456/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767048.903055:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767048.903059:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767048.903065:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967283456:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767048.903073:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967283584:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767048.903115:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767048.903131:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767048/real 1775767048] req@ffff8f9970a77480 x1862022967283584/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767048.903143:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967283584/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767048.903182:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767048.903186:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767048.903190:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967283584:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767048.903197:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967283712:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767048.903229:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767048.903241:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767048/real 1775767048] req@ffff8f9970a77100 x1862022967283712/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767048.903251:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967283712/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767048.903259:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767048.903262:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767048.903265:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967283712:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767049.797859:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767049.797929:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767050.821810:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767050.821867:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767051.845834:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767051.845888:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767052.869843:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767052.869908:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767053.893888:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767053.893976:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767054.021901:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767054.021911:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767054.021917:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767054.021928:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3675 00000100:00080000:1.0:1775767054.021939:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767054.021945:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767054.021982:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967283840/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767054.022012:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767054.022016:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767054.022018:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767054.022022:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3675 00000100:00080000:1.0:1775767054.022026:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767054.022031:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767054.022041:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767054.022049:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967283840:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767054.022056:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967283968/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767054.022068:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767054.022071:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767054.022073:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767054.022078:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3675 00000100:00080000:1.0:1775767054.022082:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767054.022086:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767054.022092:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775767054.022108:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767054.022143:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967284096/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767054.022152:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767054/real 1775767054] req@ffff8f994f2e4700 x1862022967283840/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767054.022157:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767054.022166:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767054.022169:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767054.022175:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3675 00000100:00100000:2.0:1775767054.022178:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967283840/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767054.022181:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767054.022187:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767054.022193:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1775767054.022205:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1775767054.022206:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967284224/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767054.022213:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767054.022221:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967283840:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767054.022244:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967283968:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775767054.022268:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767054.022278:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767054/real 1775767054] req@ffff8f994f2e6300 x1862022967283968/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767054.022286:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967283968/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767054.022291:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767054.022294:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767054.022296:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967283968:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767054.022300:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967284096:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767054.022312:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767054.022318:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767054/real 1775767054] req@ffff8f994f2e4e00 x1862022967284096/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767054.022325:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967284096/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767054.022330:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767054.022332:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767054.022333:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967284096:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767054.022348:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967284224:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767054.022360:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767054.022366:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767054/real 1775767054] req@ffff8f994f2e7b80 x1862022967284224/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767054.022373:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967284224/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767054.022378:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767054.022380:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767054.022382:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967284224:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767054.917883:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767054.917897:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767055.941846:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767055.941899:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767056.965846:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767056.965917:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767057.989842:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767057.989933:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767059.013823:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767059.013899:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767059.141850:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767059.141862:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767059.141868:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767059.141878:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3680 00000100:00080000:1.0:1775767059.141889:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767059.141896:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767059.141934:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967284352/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767059.141965:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767059.141969:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767059.141996:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767059.142002:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3680 00000100:00080000:1.0:1775767059.142006:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767059.142011:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767059.142015:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967284352:192.168.202.146@tcp:250:kworker.0 00000008:00100000:1.0:1775767059.142023:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767059.142044:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967284480/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767059.142057:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767059.142060:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767059.142062:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767059.142066:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3680 00000100:00080000:1.0:1775767059.142071:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775767059.142074:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775767059.142080:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767059.142086:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767059.142100:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967284608/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767059.142109:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767059.142113:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767059.142115:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767059.142119:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3680 00000100:00080000:1.0:1775767059.142123:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767059.142127:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767059.142133:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767059.142133:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767059/real 1775767059] req@ffff8f994f2e7b80 x1862022967284352/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767059.142155:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967284352/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767059.142165:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967284736/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767059.142165:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767059.142174:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767059.142182:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967284352:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767059.142220:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967284480:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775767059.142239:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767059.142248:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767059/real 1775767059] req@ffff8f994f2e4e00 x1862022967284480/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767059.142257:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967284480/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767059.142262:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767059.142264:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767059.142267:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967284480:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767059.142271:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967284608:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767059.142282:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767059.142287:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767059/real 1775767059] req@ffff8f994f2e6300 x1862022967284608/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767059.142294:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967284608/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767059.142299:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767059.142301:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767059.142303:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967284608:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767059.142306:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967284736:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767059.142315:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767059.142320:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767059/real 1775767059] req@ffff8f994f2e4700 x1862022967284736/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767059.142356:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967284736/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767059.142362:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767059.142364:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767059.142366:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967284736:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767060.037883:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767060.037955:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767061.061826:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767061.061880:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767062.085941:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767062.085965:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767063.109822:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767063.109880:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767064.133842:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767064.133907:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767064.261882:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767064.261891:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767064.261895:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767064.261904:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3685 00000100:00080000:0.0:1775767064.261914:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767064.261920:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767064.261952:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967284864/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767064.261975:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767064.262003:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767064.262004:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767064.262008:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3685 00000100:00080000:0.0:1775767064.262011:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767064.262014:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767064.262021:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967284864:192.168.202.146@tcp:250:kworker.0 00000008:00100000:0.0:1775767064.262023:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767064.262037:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967284992/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767064.262046:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767064.262048:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767064.262050:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767064.262053:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3685 00000100:00080000:0.0:1775767064.262055:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767064.262058:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767064.262061:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767064.262071:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967285120/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767064.262081:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767064.262088:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:2.0:1775767064.262088:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775767064.262095:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767064.262100:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3685 00000100:00080000:0.0:1775767064.262105:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767064.262109:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767064.262116:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767064.262129:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967285248/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767064.262138:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767064/real 1775767064] req@ffff8f9970a75500 x1862022967284864/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767064.262154:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967284864/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767064.262163:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767064.262166:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767064.262169:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967284864:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767064.262187:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967284992:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775767064.262204:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767064.262212:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767064/real 1775767064] req@ffff8f9970a76300 x1862022967284992/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767064.262239:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967284992/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767064.262244:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767064.262247:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767064.262249:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967284992:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767064.262253:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967285120:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767064.262270:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767064.262294:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767064/real 1775767064] req@ffff8f9970a75180 x1862022967285120/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767064.262309:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967285120/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767064.262318:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767064.262322:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767064.262326:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967285120:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767064.262332:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967285248:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767064.262351:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767064.262360:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767064/real 1775767064] req@ffff8f9970a77100 x1862022967285248/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767064.262368:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967285248/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767064.262391:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767064.262393:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767064.262396:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967285248:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767065.157823:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767065.157888:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767066.181811:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767066.181871:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767067.205789:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767067.205885:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767068.229846:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767068.229914:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767069.253915:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767069.253969:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767069.382399:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767069.382416:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767069.382423:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767069.382439:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3690 00000100:00080000:2.0:1775767069.382459:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767069.382471:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767069.382547:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1880 x1862022967285376/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767069.382602:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767069.382607:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767069.382609:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767069.382616:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3690 00000100:00080000:2.0:1775767069.382621:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767069.382627:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767069.382642:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767069.382658:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2a00 x1862022967285504/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767069.382932:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767069.382936:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767069.382939:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767069.382945:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3690 00000100:00080000:2.0:1775767069.382955:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767069.382961:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767069.382972:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767069.383000:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1c00 x1862022967285632/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767069.383009:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767069.383014:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767069.383016:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767069.383020:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3690 00000100:00080000:2.0:1775767069.383024:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767069.383027:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767069.383033:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767069.383046:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3b80 x1862022967285760/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767069.383071:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967285376:192.168.202.146@tcp:250:kworker.0 00000800:00000100:3.0:1775767069.383280:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767069.383340:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767069/real 1775767069] req@ffff8f99703d1880 x1862022967285376/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767069.383364:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1880 x1862022967285376/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767069.383378:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767069.383383:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767069.383387:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967285376:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767069.383395:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967285504:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767069.383419:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767069.383431:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767069/real 1775767069] req@ffff8f99703d2a00 x1862022967285504/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767069.383443:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2a00 x1862022967285504/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767069.383453:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767069.383457:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767069.383460:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967285504:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767069.383466:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967285632:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767069.383487:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767069.383499:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767069/real 1775767069] req@ffff8f99703d1c00 x1862022967285632/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767069.383511:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1c00 x1862022967285632/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767069.383520:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767069.383523:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767069.383527:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967285632:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767069.383559:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967285760:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767069.383579:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767069.383589:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767069/real 1775767069] req@ffff8f99703d3b80 x1862022967285760/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767069.383601:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3b80 x1862022967285760/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767069.383610:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767069.383614:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767069.383617:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967285760:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767070.277811:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767070.277882:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767071.301852:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767071.301919:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767072.325821:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767072.325854:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767073.349824:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767073.349909:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767074.373851:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767074.373922:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767074.501867:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767074.501878:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767074.501883:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767074.501892:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3696 00000100:00080000:0.0:1775767074.501902:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767074.501908:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767074.501940:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967285888/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767074.501965:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767074.501968:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767074.501970:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767074.501973:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3696 00000100:00080000:0.0:1775767074.501976:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767074.501979:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767074.501988:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767074.501995:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967285888:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767074.501999:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967286016/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767074.502009:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767074.502012:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767074.502014:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767074.502017:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3696 00000100:00080000:0.0:1775767074.502019:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767074.502021:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767074.502025:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767074.502033:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967286144/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767074.502039:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767074.502041:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767074.502043:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767074.502046:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3696 00000100:00080000:0.0:1775767074.502049:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767074.502051:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767074.502054:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775767074.502059:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767074.502062:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967286272/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767074.502126:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767074/real 1775767074] req@ffff8f9970a77480 x1862022967285888/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767074.502142:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967285888/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767074.502150:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767074.502153:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767074.502156:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967285888:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767074.502205:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967286016:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767074.502227:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767074.502236:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767074/real 1775767074] req@ffff8f9970a75c00 x1862022967286016/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767074.502246:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967286016/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767074.502252:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767074.502254:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767074.502256:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967286016:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767074.502260:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967286144:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767074.502284:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767074.502290:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767074/real 1775767074] req@ffff8f9970a76a00 x1862022967286144/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767074.502298:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967286144/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767074.502303:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767074.502305:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767074.502307:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967286144:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767074.502311:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967286272:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767074.502320:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767074.502325:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767074/real 1775767074] req@ffff8f9970a74380 x1862022967286272/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767074.502332:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967286272/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767074.502342:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767074.502344:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767074.502346:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967286272:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767075.397850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767075.397947:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767076.421902:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767076.421912:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767077.445831:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767077.445920:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767078.469810:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767078.469889:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767079.493831:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767079.493915:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767079.621829:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767079.621839:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767079.621844:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767079.621854:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3701 00000100:00080000:0.0:1775767079.621866:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767079.621874:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767079.621913:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967286400/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767079.621943:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767079.621946:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767079.621948:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775767079.621970:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967286400:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775767079.621989:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3701 00000100:00080000:0.0:1775767079.621993:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767079.621997:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767079.622007:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767079.622033:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967286528/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767079.622046:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767079.622049:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767079.622052:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1775767079.622057:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775767079.622060:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3701 00000100:00080000:0.0:1775767079.622066:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767079.622071:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767079.622077:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767079.622090:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967286656/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767079.622101:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767079.622104:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1775767079.622114:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767079/real 1775767079] req@ffff8f9970a77800 x1862022967286400/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767079.622143:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767079.622154:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3701 00000100:00080000:0.0:1775767079.622160:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767079.622164:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767079.622171:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767079.622171:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967286400/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767079.622185:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767079.622190:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767079.622195:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967286400:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767079.622202:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967286784/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767079.622223:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967286528:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767079.622244:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767079.622254:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767079/real 1775767079] req@ffff8f9970a76d80 x1862022967286528/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767079.622263:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967286528/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767079.622269:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767079.622271:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767079.622273:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967286528:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767079.622278:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967286656:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767079.622289:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767079.622295:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767079/real 1775767079] req@ffff8f9970a74700 x1862022967286656/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767079.622302:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967286656/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767079.622308:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767079.622310:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767079.622311:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967286656:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767079.622315:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967286784:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767079.622334:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767079.622340:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767079/real 1775767079] req@ffff8f9970a74380 x1862022967286784/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767079.622348:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967286784/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767079.622364:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767079.622366:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767079.622368:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967286784:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767080.517827:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767080.517882:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767081.541836:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767081.541881:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767082.565864:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767082.565903:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767083.589881:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767083.589927:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775767084.613831:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767084.613869:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767084.741928:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767084.741936:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767084.741941:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767084.741952:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3706 00000100:00080000:2.0:1775767084.741962:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767084.741987:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767084.742033:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3b80 x1862022967286912/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767084.742057:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767084.742060:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767084.742061:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767084.742081:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3706 00000100:00080000:2.0:1775767084.742086:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767084.742092:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767084.742120:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767084.742139:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967286912:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767084.742159:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1c00 x1862022967287040/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767084.742191:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767084.742195:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767084.742197:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767084.742201:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3706 00000100:00080000:2.0:1775767084.742206:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767084.742210:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767084.742216:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767084.742233:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2a00 x1862022967287168/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767084.742234:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775767084.742249:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767084.742254:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767084.742280:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767084.742284:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3706 00000100:00080000:2.0:1775767084.742287:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767084.742290:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767084.742297:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767084.742312:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767084/real 1775767084] req@ffff8f99703d3b80 x1862022967286912/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767084.742335:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3b80 x1862022967286912/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767084.742345:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1880 x1862022967287296/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767084.742346:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767084.742356:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767084.742362:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967286912:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767084.742388:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967287040:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775767084.742415:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767084.742429:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767084/real 1775767084] req@ffff8f99703d1c00 x1862022967287040/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767084.742442:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1c00 x1862022967287040/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767084.742452:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767084.742455:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767084.742459:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967287040:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767084.742465:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967287168:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767084.742485:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767084.742493:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767084/real 1775767084] req@ffff8f99703d2a00 x1862022967287168/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767084.742501:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2a00 x1862022967287168/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767084.742507:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767084.742520:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767084.742524:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967287168:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767084.742531:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967287296:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767084.742554:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767084.742570:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767084/real 1775767084] req@ffff8f99703d1880 x1862022967287296/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767084.742586:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1880 x1862022967287296/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767084.742592:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767084.742595:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767084.742614:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967287296:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767085.637863:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767085.637930:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767086.661878:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767086.661952:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767087.685831:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767087.685928:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767088.709832:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767088.709894:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767089.733811:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767089.733851:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767089.861825:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767089.861834:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767089.861838:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767089.861846:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3711 00000100:00080000:2.0:1775767089.861855:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767089.861861:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767089.861895:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3480 x1862022967287424/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767089.861919:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767089.861922:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767089.861923:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767089.861926:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3711 00000100:00080000:2.0:1775767089.861928:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767089.861930:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767089.861938:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767089.861945:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1180 x1862022967287552/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767089.861952:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767089.861955:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767089.861956:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767089.861959:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3711 00000100:00080000:2.0:1775767089.861961:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767089.861963:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767089.861966:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767089.861978:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967287424:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767089.862029:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2300 x1862022967287680/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767089.862036:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767089.862039:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767089.862040:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767089.862042:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3711 00000100:00080000:2.0:1775767089.862046:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767089.862049:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767089.862053:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767089.862061:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3100 x1862022967287808/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767089.862073:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767089.862112:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767089/real 1775767089] req@ffff8f99703d3480 x1862022967287424/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767089.862131:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3480 x1862022967287424/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767089.862140:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767089.862145:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767089.862149:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967287424:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767089.862157:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967287552:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775767089.862191:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767089.862206:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767089/real 1775767089] req@ffff8f99703d1180 x1862022967287552/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767089.862220:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1180 x1862022967287552/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767089.862228:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767089.862232:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767089.862235:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967287552:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767089.862260:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967287680:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767089.862283:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767089.862294:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767089/real 1775767089] req@ffff8f99703d2300 x1862022967287680/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767089.862307:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2300 x1862022967287680/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767089.862316:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767089.862320:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767089.862323:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967287680:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767089.862330:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967287808:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767089.862349:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767089.862359:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767089/real 1775767089] req@ffff8f99703d3100 x1862022967287808/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767089.862372:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3100 x1862022967287808/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767089.862380:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767089.862383:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767089.862387:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967287808:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767090.757971:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767090.758032:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767091.781856:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767091.781940:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767092.805835:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767092.805929:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767093.829831:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767093.829909:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767094.853870:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767094.854000:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767094.981880:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767094.981894:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767094.981899:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767094.982030:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3716 00000100:00080000:0.0:1775767094.982041:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767094.982048:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767094.982100:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967287936/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767094.982125:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767094.982129:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767094.982131:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767094.982134:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3716 00000100:00080000:0.0:1775767094.982137:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767094.982140:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767094.982148:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767094.982159:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967287936:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767094.982169:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967288064/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767094.982178:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767094.982180:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767094.982181:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767094.982184:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3716 00000100:00080000:0.0:1775767094.982186:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767094.982188:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767094.982192:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767094.982206:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967288192/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767094.982217:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767094.982219:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767094.982221:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767094.982223:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3716 00000100:00080000:0.0:1775767094.982225:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767094.982227:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767094.982229:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767094.982235:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967288320/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775767094.982288:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767094.982332:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767094/real 1775767094] req@ffff8f9970a76a00 x1862022967287936/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767094.982349:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967287936/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767094.982358:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767094.982362:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767094.982365:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967287936:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767094.982382:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967288064:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767094.982398:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767094.982413:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767094/real 1775767094] req@ffff8f9970a75c00 x1862022967288064/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767094.982422:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967288064/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767094.982427:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767094.982429:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767094.982432:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967288064:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767094.982435:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967288192:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767094.982448:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767094.982454:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767094/real 1775767094] req@ffff8f9970a77480 x1862022967288192/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767094.982463:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967288192/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767094.982477:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767094.982481:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767094.982484:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967288192:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767094.982490:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967288320:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767094.982509:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767094.982522:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767094/real 1775767094] req@ffff8f9970a77100 x1862022967288320/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767094.982533:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967288320/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767094.982542:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767094.982545:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767094.982548:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967288320:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767095.877838:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767095.877910:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767096.901844:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767096.901925:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767097.925889:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767097.925890:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767098.949860:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767098.949960:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767099.973865:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767099.974128:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767100.101806:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767100.101817:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767100.101822:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767100.101832:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3721 00000100:00080000:0.0:1775767100.101843:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767100.101850:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767100.101888:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967288448/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767100.101935:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767100.101940:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767100.101942:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767100.101948:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3721 00000100:00080000:0.0:1775767100.101954:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775767100.101957:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967288448:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775767100.101962:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767100.101974:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767100.101989:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967288576/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767100.101998:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767100.102000:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767100.102002:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767100.102005:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3721 00000100:00080000:0.0:1775767100.102008:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767100.102011:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767100.102015:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775767100.102016:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767100.102026:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967288704/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767100.102034:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767100.102036:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767100.102037:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767100.102040:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3721 00000100:00080000:0.0:1775767100.102042:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767100.102044:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767100.102048:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767100.102056:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967288832/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767100.102063:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767100/real 1775767100] req@ffff8f9970a75180 x1862022967288448/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767100.102119:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967288448/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767100.102131:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767100.102135:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767100.102139:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967288448:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767100.102164:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967288576:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767100.102189:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767100.102202:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767100/real 1775767100] req@ffff8f9970a76300 x1862022967288576/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767100.102212:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967288576/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767100.102217:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767100.102220:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767100.102222:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967288576:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767100.102226:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967288704:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767100.102240:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767100.102246:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767100/real 1775767100] req@ffff8f9970a75500 x1862022967288704/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767100.102256:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967288704/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767100.102262:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767100.102264:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767100.102267:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967288704:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767100.102271:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967288832:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767100.102282:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767100.102287:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767100/real 1775767100] req@ffff8f9970a77100 x1862022967288832/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767100.102295:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967288832/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767100.102300:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767100.102303:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767100.102305:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967288832:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767100.997836:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767100.997908:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767102.021803:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767102.021902:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767103.045829:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767103.045887:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767104.069813:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767104.069925:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767105.093890:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767105.093898:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767105.221822:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767105.221830:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767105.221834:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767105.221842:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3726 00000100:00080000:0.0:1775767105.221863:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767105.221868:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767105.221901:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967288960/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767105.221924:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767105.221927:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767105.221928:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767105.221931:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3726 00000100:00080000:0.0:1775767105.221934:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767105.221937:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767105.221944:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767105.221955:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967289088/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767105.221957:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967288960:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775767105.221970:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767105.221972:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767105.221973:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767105.221976:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3726 00000100:00080000:0.0:1775767105.221983:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767105.221985:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767105.221990:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767105.221998:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967289216/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767105.222004:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767105.222006:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767105.222007:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767105.222010:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3726 00000100:00080000:0.0:1775767105.222012:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767105.222014:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767105.222017:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775767105.222021:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767105.222063:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967289344/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767105.222069:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767105/real 1775767105] req@ffff8f9970a77480 x1862022967288960/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767105.222087:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967288960/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767105.222095:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767105.222099:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767105.222103:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967288960:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767105.222121:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967289088:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767105.222136:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767105.222144:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767105/real 1775767105] req@ffff8f9970a75c00 x1862022967289088/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767105.222152:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967289088/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767105.222162:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767105.222164:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767105.222166:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967289088:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767105.222169:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967289216:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767105.222179:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767105.222185:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767105/real 1775767105] req@ffff8f9970a76a00 x1862022967289216/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767105.222192:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967289216/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767105.222197:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767105.222198:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767105.222200:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967289216:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767105.222204:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967289344:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767105.222213:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767105.222218:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767105/real 1775767105] req@ffff8f9970a74380 x1862022967289344/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767105.222225:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967289344/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767105.222230:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767105.222232:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767105.222234:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967289344:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767106.117814:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767106.117888:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767107.141914:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767107.141985:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767108.165853:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767108.165910:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767109.189832:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767109.189905:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767110.213883:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767110.213913:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767110.341848:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767110.341856:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767110.341860:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767110.341869:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3731 00000100:00080000:2.0:1775767110.341880:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767110.341885:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767110.341917:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1500 x1862022967289472/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767110.341938:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767110.341941:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767110.341942:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767110.341945:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3731 00000100:00080000:2.0:1775767110.341947:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767110.341950:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767110.341957:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767110.341965:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0700 x1862022967289600/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767110.341972:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767110.341975:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767110.341976:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767110.341979:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3731 00000100:00080000:2.0:1775767110.341981:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775767110.341985:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967289472:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775767110.342012:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767110.342017:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767110.342026:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0e00 x1862022967289728/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767110.342033:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767110.342035:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767110.342037:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767110.342040:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3731 00000100:00080000:2.0:1775767110.342042:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767110.342044:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767110.342047:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767110.342054:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2d80 x1862022967289856/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767110.342096:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967289600:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767110.342137:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967289728:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767110.342185:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967289856:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767111.237837:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767111.237892:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767112.261902:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767112.261924:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767113.285872:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767113.285958:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775767113.414593:0:2369:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:3.0:1775767113.414682:0:2369:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775767113.414710:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:3.0:1775767113.414847:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:3.0:1775767113.414870:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:3.0:1775767113.414887:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:1.0:1775767113.415135:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767110/real 1775767113] req@ffff8f99703d0e00 x1862022967289728/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767113.415173:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0e00 x1862022967289728/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767113.415187:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767113.415193:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767113.415198:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967289728:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767113.415208:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767110/real 1775767113] req@ffff8f99703d2d80 x1862022967289856/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767113.415221:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2d80 x1862022967289856/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767113.415228:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767113.415238:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767113.415242:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967289856:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767113.415250:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767110/real 1775767113] req@ffff8f99703d1500 x1862022967289472/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767113.415263:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1500 x1862022967289472/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767113.415271:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767113.415276:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767113.415302:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967289472:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767113.415310:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767110/real 1775767113] req@ffff8f99703d0700 x1862022967289600/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767113.415329:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0700 x1862022967289600/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767113.415349:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767113.415354:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767113.415358:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967289600:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767114.309829:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767114.309889:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767115.333849:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767115.333914:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767115.461890:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767115.461898:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767115.461902:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767115.461922:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3737 00000100:00080000:3.0:1775767115.461932:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767115.461937:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767115.462018:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967289984/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767115.462055:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767115.462059:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767115.462061:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767115.462067:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3737 00000100:00080000:3.0:1775767115.462070:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767115.462074:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767115.462084:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767115.462108:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967289984:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767115.462116:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967290112/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767115.462130:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767115.462133:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767115.462135:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767115.462139:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3737 00000100:00080000:3.0:1775767115.462142:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767115.462146:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767115.462151:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767115.462172:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967290240/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767115.462182:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767115.462186:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767115.462187:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767115.462192:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3737 00000100:00080000:3.0:1775767115.462196:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767115.462199:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767115.462204:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767115.462236:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967290368/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767115.462255:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767115.462306:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767115/real 1775767115] req@ffff8f9970edfb80 x1862022967289984/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767115.462327:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967289984/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767115.462338:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767115.462342:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767115.462347:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967289984:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767115.462373:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967290112:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775767115.462398:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767115.462412:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767115/real 1775767115] req@ffff8f9970edc380 x1862022967290112/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767115.462444:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967290112/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767115.462453:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767115.462457:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767115.462460:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967290112:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767115.462467:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967290240:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767115.462504:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767115.462517:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767115/real 1775767115] req@ffff8f9970edca80 x1862022967290240/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767115.462529:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967290240/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767115.462536:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767115.462539:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767115.462542:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967290240:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767115.462548:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967290368:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767115.462568:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767115.462579:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767115/real 1775767115] req@ffff8f9970edea00 x1862022967290368/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767115.462590:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967290368/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767115.462618:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767115.462622:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767115.462625:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967290368:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767116.357855:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767116.357911:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767117.381837:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767117.381893:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767118.405832:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767118.405940:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767119.429867:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767119.429913:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767120.453816:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767120.453870:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767120.581819:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767120.581827:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767120.581832:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767120.581841:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3742 00000100:00080000:3.0:1775767120.581850:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767120.581857:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767120.581917:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd180 x1862022967290496/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767120.581941:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767120.581944:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767120.581946:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767120.581949:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3742 00000100:00080000:3.0:1775767120.581953:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767120.581957:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767120.581966:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767120.581975:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967290496:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767120.581980:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddf80 x1862022967290624/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767120.581990:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767120.581993:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767120.581994:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767120.581997:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3742 00000100:00080000:3.0:1775767120.582000:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767120.582003:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767120.582017:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767120.582026:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967290752/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767120.582032:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767120.582035:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767120.582036:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767120.582039:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3742 00000800:00000100:2.0:1775767120.582062:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775767120.582115:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767120.582121:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767120.582127:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767120.582144:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767120/real 1775767120] req@ffff8f9970edd180 x1862022967290496/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767120.582172:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd180 x1862022967290496/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767120.582180:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767120.582183:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767120.582186:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967290496:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767120.582206:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967290624:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775767120.582225:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767120.582237:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767120/real 1775767120] req@ffff8f9970eddf80 x1862022967290624/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767120.582240:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967290880/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767120.582264:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddf80 x1862022967290624/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767120.582270:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767120.582273:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767120.582275:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967290624:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767120.582280:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967290752:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767120.582296:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767120.582312:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767120/real 1775767120] req@ffff8f9970edea00 x1862022967290752/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767120.582326:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967290752/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767120.582348:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767120.582350:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767120.582353:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967290752:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767120.582363:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967290880:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767120.582375:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767120.582381:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767120/real 1775767120] req@ffff8f9970edca80 x1862022967290880/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767120.582389:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967290880/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767120.582394:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767120.582396:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767120.582399:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967290880:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767121.477794:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767121.477875:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767122.501821:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767122.501898:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767123.525821:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767123.525894:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767124.549816:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767124.549895:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767125.573906:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767125.573997:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767125.701863:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767125.701876:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767125.701882:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767125.701893:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3747 00000100:00080000:2.0:1775767125.701906:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767125.701914:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767125.701970:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3800 x1862022967291008/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767125.702080:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767125.702085:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767125.702088:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767125.702094:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3747 00000100:00080000:2.0:1775767125.702100:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767125.702104:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767125.702115:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767125.702133:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0380 x1862022967291136/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767125.702145:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767125.702149:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767125.702151:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767125.702156:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3747 00000100:00080000:2.0:1775767125.702160:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767125.702164:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767125.702169:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767125.702185:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967291008:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767125.702185:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0000 x1862022967291264/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767125.702208:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767125.702211:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767125.702213:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767125.702217:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3747 00000100:00080000:2.0:1775767125.702242:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767125.702247:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767125.702252:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767125.702265:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2680 x1862022967291392/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767125.702279:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767125.702343:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767125/real 1775767125] req@ffff8f99703d3800 x1862022967291008/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767125.702364:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3800 x1862022967291008/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767125.702376:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767125.702381:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767125.702385:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967291008:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767125.702394:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967291136:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775767125.702416:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767125.702428:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767125/real 1775767125] req@ffff8f99703d0380 x1862022967291136/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767125.702449:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0380 x1862022967291136/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767125.702458:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767125.702461:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767125.702464:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967291136:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767125.702489:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967291264:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767125.702507:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767125.702517:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767125/real 1775767125] req@ffff8f99703d0000 x1862022967291264/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767125.702533:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0000 x1862022967291264/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767125.702556:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767125.702562:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767125.702567:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967291264:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767125.702576:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967291392:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767125.702607:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767125.702635:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767125/real 1775767125] req@ffff8f99703d2680 x1862022967291392/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767125.702701:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2680 x1862022967291392/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767125.702717:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767125.702721:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767125.702724:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967291392:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767126.597813:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767126.597871:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767127.621840:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767127.621949:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767128.645842:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767128.645884:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767129.669837:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767129.670010:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767130.693815:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767130.693882:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767130.821840:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767130.821850:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767130.821855:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767130.821866:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3752 00000100:00080000:2.0:1775767130.821877:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767130.821884:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767130.821924:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2680 x1862022967291520/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767130.821946:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767130.821951:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767130.821959:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767130.821964:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3752 00000100:00080000:2.0:1775767130.821968:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767130.821983:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767130.821987:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967291520:192.168.202.146@tcp:250:kworker.0 00000008:00100000:2.0:1775767130.821993:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767130.822004:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0000 x1862022967291648/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767130.822014:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1775767130.822056:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775767130.822058:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767130.822059:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767130.822063:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3752 00000100:00080000:2.0:1775767130.822066:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767130.822070:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767130.822084:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767130.822094:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0380 x1862022967291776/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767130.822104:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767130.822107:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767130.822109:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767130.822112:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3752 00000100:00100000:1.0:1775767130.822115:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767130/real 1775767130] req@ffff8f99703d2680 x1862022967291520/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767130.822115:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767130.822119:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767130.822124:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767130.822133:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3800 x1862022967291904/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767130.822134:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2680 x1862022967291520/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767130.822145:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767130.822149:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767130.822154:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967291520:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767130.822176:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967291648:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775767130.822196:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767130.822206:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767130/real 1775767130] req@ffff8f99703d0000 x1862022967291648/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767130.822214:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0000 x1862022967291648/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767130.822220:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767130.822222:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767130.822224:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967291648:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767130.822228:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967291776:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767130.822239:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767130.822245:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767130/real 1775767130] req@ffff8f99703d0380 x1862022967291776/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767130.822252:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0380 x1862022967291776/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767130.822257:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767130.822258:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767130.822260:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967291776:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767130.822263:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967291904:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767130.822273:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767130.822278:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767130/real 1775767130] req@ffff8f99703d3800 x1862022967291904/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767130.822285:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3800 x1862022967291904/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767130.822290:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767130.822292:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767130.822294:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967291904:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767131.717841:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767131.717904:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767132.741819:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767132.741923:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767133.765863:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767133.765933:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767134.789821:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767134.789917:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767135.813852:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767135.813914:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767135.942220:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767135.942232:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767135.942237:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767135.942248:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3757 00000100:00080000:3.0:1775767135.942262:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767135.942269:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767135.942318:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967292032/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767135.942362:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767135.942366:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767135.942368:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767135.942373:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3757 00000100:00080000:3.0:1775767135.942413:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767135.942421:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767135.942422:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967292032:192.168.202.146@tcp:250:kworker.0 00000008:00100000:3.0:1775767135.942437:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767135.942455:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967292160/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767135.942467:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767135.942470:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767135.942472:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767135.942476:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3757 00000100:00080000:3.0:1775767135.942480:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767135.942484:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767135.942489:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767135.942519:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967292288/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767135.942529:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767135.942544:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767135.942545:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767135.942549:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3757 00000100:00080000:3.0:1775767135.942554:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775767135.942560:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775767135.942562:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767135.942569:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767135.942581:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967292416/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767135.942607:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767135/real 1775767135] req@ffff8f9970edc380 x1862022967292032/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767135.942640:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967292032/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767135.942652:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767135.942658:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767135.942664:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967292032:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767135.942688:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967292160:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775767135.942710:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767135.942723:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767135/real 1775767135] req@ffff8f9970edfb80 x1862022967292160/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767135.942735:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967292160/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767135.942785:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767135.942788:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767135.942791:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967292160:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767135.942802:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967292288:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767135.942822:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767135.942831:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767135/real 1775767135] req@ffff8f9970ede300 x1862022967292288/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767135.942839:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967292288/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767135.942844:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767135.942846:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767135.942849:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967292288:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767135.942852:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967292416:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767135.942863:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767135.942868:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767135/real 1775767135] req@ffff8f9970edd880 x1862022967292416/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767135.942876:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967292416/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767135.942881:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767135.942882:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767135.942884:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967292416:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767136.837841:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767136.837898:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767137.861829:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767137.861944:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767138.885884:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767138.885940:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767139.909860:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767139.909975:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767140.933840:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767140.933889:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767141.061897:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767141.061927:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767141.061932:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767141.061943:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3762 00000100:00080000:2.0:1775767141.061956:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767141.061963:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767141.062004:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0700 x1862022967292544/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767141.062094:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767141.062097:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767141.062099:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767141.062104:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3762 00000100:00080000:2.0:1775767141.062107:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767141.062111:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767141.062121:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767141.062149:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1500 x1862022967292672/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767141.062157:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775767141.062160:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967292544:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775767141.062160:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767141.062164:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767141.062171:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3762 00000100:00080000:2.0:1775767141.062173:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767141.062176:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767141.062180:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767141.062189:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2d80 x1862022967292800/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767141.062196:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767141.062199:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767141.062200:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767141.062202:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3762 00000100:00080000:2.0:1775767141.062215:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767141.062218:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:0.0:1775767141.062223:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:2.0:1775767141.062225:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767141.062236:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0e00 x1862022967292928/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767141.062273:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767141/real 1775767141] req@ffff8f99703d0700 x1862022967292544/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767141.062294:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0700 x1862022967292544/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767141.062304:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767141.062308:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767141.062311:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967292544:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767141.062329:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967292672:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775767141.062347:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767141.062356:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767141/real 1775767141] req@ffff8f99703d1500 x1862022967292672/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767141.062364:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1500 x1862022967292672/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767141.062370:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767141.062372:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767141.062374:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967292672:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767141.062378:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967292800:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767141.062390:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767141.062396:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767141/real 1775767141] req@ffff8f99703d2d80 x1862022967292800/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767141.062403:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2d80 x1862022967292800/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767141.062408:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767141.062410:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767141.062412:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967292800:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767141.062415:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967292928:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767141.062444:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767141.062452:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767141/real 1775767141] req@ffff8f99703d0e00 x1862022967292928/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767141.062462:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0e00 x1862022967292928/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767141.062469:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767141.062471:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767141.062474:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967292928:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767141.957950:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767141.957999:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767142.981810:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767142.981864:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767144.005855:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767144.005880:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767145.029900:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767145.029957:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767146.053842:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767146.053863:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767146.181843:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767146.181853:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767146.181858:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767146.181868:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3767 00000100:00080000:3.0:1775767146.181878:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767146.181885:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767146.181921:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967293056/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767146.181949:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767146.181953:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767146.181955:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767146.181959:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3767 00000100:00080000:3.0:1775767146.181963:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767146.181967:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767146.181977:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767146.181991:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967293184/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767146.181994:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967293056:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775767146.182013:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767146.182017:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767146.182019:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767146.182025:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3767 00000100:00080000:3.0:1775767146.182028:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767146.182032:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:0.0:1775767146.182054:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:3.0:1775767146.182077:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767146.182088:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767146/real 1775767146] req@ffff8f9970edf480 x1862022967293056/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767146.182095:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967293312/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767146.182108:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967293056/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767146.182109:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767146.182119:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767146.182123:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767146.182126:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767146.182130:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3767 00000100:00080000:0.0:1775767146.182134:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00080000:3.0:1775767146.182138:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775767146.182139:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967293056:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775767146.182148:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767146.182153:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767146.182163:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967293184:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767146.182169:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967293440/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767146.182181:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767146.182193:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767146/real 1775767146] req@ffff8f9970edf800 x1862022967293184/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767146.182212:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967293184/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767146.182218:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767146.182220:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767146.182222:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967293184:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767146.182226:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967293312:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767146.182239:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767146.182249:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767146/real 1775767146] req@ffff8f9970edc000 x1862022967293312/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767146.182262:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967293312/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767146.182267:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767146.182269:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767146.182270:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967293312:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767146.182279:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967293440:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767146.182291:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767146.182300:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767146/real 1775767146] req@ffff8f9970ede680 x1862022967293440/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767146.182312:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967293440/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767146.182320:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767146.182324:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767146.182327:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967293440:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767147.077868:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767147.077942:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767148.101897:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767148.101943:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767149.125905:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767149.125932:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767150.149814:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767150.149974:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767151.173825:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767151.173900:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767151.301842:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767151.301853:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767151.301858:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767151.301870:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3772 00000100:00080000:3.0:1775767151.301883:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767151.301890:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767151.301924:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967293568/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767151.301951:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767151.301954:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1775767151.301986:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967293568:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775767151.301991:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767151.301999:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3772 00000100:00080000:3.0:1775767151.302003:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767151.302006:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767151.302015:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767151.302030:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967293696/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767151.302040:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767151.302043:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767151.302045:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767151.302057:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3772 00000800:00000100:0.0:1775767151.302062:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775767151.302062:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767151.302070:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767151.302075:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767151.302087:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967293824/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767151.302095:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767151.302098:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767151.302100:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767151.302103:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3772 00000100:00080000:3.0:1775767151.302105:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767151.302108:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767151.302112:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767151.302115:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767151/real 1775767151] req@ffff8f9970edd500 x1862022967293568/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767151.302141:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967293952/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767151.302149:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967293568/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767151.302164:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767151.302168:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767151.302174:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967293568:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767151.302197:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967293696:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775767151.302227:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767151.302237:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767151/real 1775767151] req@ffff8f9970eded80 x1862022967293696/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767151.302247:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967293696/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767151.302253:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767151.302256:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767151.302258:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967293696:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767151.302263:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967293824:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767151.302276:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767151.302283:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767151/real 1775767151] req@ffff8f9970edce00 x1862022967293824/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767151.302303:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967293824/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767151.302347:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767151.302352:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767151.302357:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967293824:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767151.302365:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967293952:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767151.302411:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767151.302423:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767151/real 1775767151] req@ffff8f9970edc700 x1862022967293952/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767151.302436:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967293952/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767151.302446:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767151.302451:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767151.302455:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967293952:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767152.197778:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767152.197918:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767153.221840:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767153.221904:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767154.245893:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767154.246024:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767155.269883:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767155.269928:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767156.293825:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767156.293908:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767156.421826:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767156.421838:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767156.421843:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767156.421855:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3778 00000100:00080000:3.0:1775767156.421867:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767156.421874:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767156.421905:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddc00 x1862022967294080/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767156.421927:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767156.421930:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767156.421932:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767156.421936:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3778 00000100:00080000:3.0:1775767156.421939:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767156.421942:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767156.421948:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967294080:192.168.202.146@tcp:250:kworker.0 00000008:00100000:3.0:1775767156.421951:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767156.421964:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf100 x1862022967294208/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767156.421972:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767156.421974:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767156.421976:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767156.421979:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3778 00000100:00080000:3.0:1775767156.421982:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775767156.422010:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775767156.422025:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767156.422030:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767156.422041:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967294336/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767156.422048:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767156/real 1775767156] req@ffff8f9970eddc00 x1862022967294080/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767156.422050:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767156.422057:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767156.422061:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767156.422068:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3778 00000100:00080000:3.0:1775767156.422073:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767156.422076:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767156.422079:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddc00 x1862022967294080/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1775767156.422081:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1775767156.422093:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1775767156.422097:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967294464/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767156.422099:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767156.422108:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967294080:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767156.422136:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967294208:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775767156.422165:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767156.422181:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767156/real 1775767156] req@ffff8f9970edf100 x1862022967294208/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767156.422215:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf100 x1862022967294208/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767156.422221:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767156.422223:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767156.422225:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967294208:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767156.422230:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967294336:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767156.422245:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767156.422252:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767156/real 1775767156] req@ffff8f9970edc700 x1862022967294336/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767156.422260:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967294336/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767156.422264:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767156.422266:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767156.422268:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967294336:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767156.422272:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967294464:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767156.422281:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767156.422287:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767156/real 1775767156] req@ffff8f9970edce00 x1862022967294464/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767156.422294:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967294464/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767156.422327:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767156.422331:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767156.422333:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967294464:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767157.317818:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767157.317888:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767158.341817:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767158.341862:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767159.365822:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767159.365875:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767160.389804:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767160.389860:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767161.413825:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767161.413910:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767161.541865:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767161.541874:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767161.541878:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767161.541887:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3783 00000100:00080000:2.0:1775767161.541896:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767161.541903:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767161.541936:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3100 x1862022967294592/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767161.541963:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767161.541966:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767161.541967:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767161.541971:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3783 00000100:00080000:2.0:1775767161.541973:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767161.541975:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767161.541982:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767161.542001:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2300 x1862022967294720/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767161.542047:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775767161.542049:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967294592:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775767161.542050:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767161.542052:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767161.542069:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3783 00000100:00080000:2.0:1775767161.542072:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767161.542074:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767161.542077:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767161.542086:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1180 x1862022967294848/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767161.542093:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767161.542095:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767161.542097:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767161.542099:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3783 00000100:00080000:2.0:1775767161.542101:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767161.542103:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767161.542106:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767161.542114:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3480 x1862022967294976/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767161.542134:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767161.542197:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767161/real 1775767161] req@ffff8f99703d3100 x1862022967294592/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767161.542229:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3100 x1862022967294592/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767161.542241:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767161.542291:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767161.542296:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967294592:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767161.542321:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967294720:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775767161.542346:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767161.542372:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767161/real 1775767161] req@ffff8f99703d2300 x1862022967294720/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767161.542385:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2300 x1862022967294720/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767161.542394:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767161.542397:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767161.542401:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967294720:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767161.542408:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967294848:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767161.542430:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767161.542441:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767161/real 1775767161] req@ffff8f99703d1180 x1862022967294848/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767161.542453:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1180 x1862022967294848/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767161.542462:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767161.542465:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767161.542468:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967294848:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767161.542474:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967294976:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767161.542513:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767161.542525:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767161/real 1775767161] req@ffff8f99703d3480 x1862022967294976/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767161.542537:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3480 x1862022967294976/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767161.542545:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767161.542548:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767161.542551:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967294976:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767162.437894:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767162.437939:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767163.461887:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767163.461963:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767164.485836:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767164.485905:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767165.509837:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767165.509933:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767166.533881:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767166.533900:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767166.661864:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767166.661873:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767166.661877:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767166.661889:0:8:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3788 00000100:00080000:1.0:1775767166.661901:0:8:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767166.661907:0:8:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767166.661974:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967295104/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767166.662002:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767166.662005:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767166.662006:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767166.662009:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3788 00000100:00080000:1.0:1775767166.662012:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767166.662014:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767166.662022:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767166.662030:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967295232/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767166.662037:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767166.662039:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767166.662040:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767166.662043:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3788 00000100:00080000:1.0:1775767166.662044:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767166.662046:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767166.662050:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767166.662124:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967295360/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767166.662128:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967295104:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775767166.662147:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767166.662149:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767166.662151:0:8:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767166.662154:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3788 00000100:00080000:1.0:1775767166.662156:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767166.662160:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767166.662164:0:8:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767166.662183:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967295488/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767166.662238:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767166.662294:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767166/real 1775767166] req@ffff8f994f2e6a00 x1862022967295104/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767166.662314:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967295104/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767166.662323:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767166.662334:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767166.662337:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967295104:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767166.662344:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967295232:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775767166.662360:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767166.662369:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767166/real 1775767166] req@ffff8f994f2e5f80 x1862022967295232/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767166.662467:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967295232/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767166.662473:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767166.662476:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767166.662478:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967295232:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767166.662507:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967295360:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767166.662529:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767166.662536:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767166/real 1775767166] req@ffff8f994f2e4000 x1862022967295360/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767166.662544:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967295360/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767166.662550:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767166.662552:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767166.662554:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967295360:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767166.662558:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967295488:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767166.662568:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767166.662573:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767166/real 1775767166] req@ffff8f994f2e6680 x1862022967295488/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767166.662580:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967295488/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767166.662603:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767166.662606:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767166.662608:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967295488:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767167.557825:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767167.557885:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767168.581904:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767168.582030:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767169.605876:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767169.605877:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775767170.629872:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767170.629953:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767171.653801:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767171.653866:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767171.781875:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767171.781883:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767171.781886:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767171.781895:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3793 00000100:00080000:0.0:1775767171.781905:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767171.781910:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767171.781942:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967295616/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767171.781966:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767171.781969:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767171.781970:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767171.781974:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3793 00000100:00080000:0.0:1775767171.781976:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767171.781979:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767171.781988:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767171.781996:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967295616:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767171.781998:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967295744/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767171.782011:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767171.782050:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767171.782052:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767171.782055:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3793 00000100:00080000:0.0:1775767171.782058:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767171.782060:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767171.782064:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775767171.782065:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767171.782076:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967295872/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767171.782095:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767171.782097:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767171.782099:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767171.782102:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3793 00000100:00080000:0.0:1775767171.782105:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767171.782107:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767171.782110:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767171.782121:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767171/real 1775767171] req@ffff8f9970a74700 x1862022967295616/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767171.782123:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967296000/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767171.782183:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967295616/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767171.782193:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767171.782196:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767171.782199:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967295616:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767171.782218:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967295744:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775767171.782236:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767171.782245:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767171/real 1775767171] req@ffff8f9970a76d80 x1862022967295744/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767171.782254:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967295744/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767171.782259:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767171.782261:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767171.782263:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967295744:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767171.782286:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967295872:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767171.782299:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767171.782306:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767171/real 1775767171] req@ffff8f9970a77800 x1862022967295872/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767171.782314:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967295872/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767171.782320:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767171.782322:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767171.782324:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967295872:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767171.782327:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967296000:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767171.782337:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767171.782342:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767171/real 1775767171] req@ffff8f9970a74380 x1862022967296000/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767171.782349:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967296000/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767171.782361:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767171.782363:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767171.782365:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967296000:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767172.677844:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767172.677942:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767173.701876:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767173.701985:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767174.725871:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767174.725935:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767175.749810:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767175.749917:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767176.773885:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767176.773897:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767176.901865:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767176.901877:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767176.901881:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767176.901892:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3798 00000100:00080000:0.0:1775767176.901904:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767176.901910:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767176.901984:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967296128/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767176.902013:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767176.902016:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767176.902018:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767176.902023:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3798 00000100:00080000:0.0:1775767176.902028:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767176.902032:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767176.902042:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767176.902046:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967296128:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767176.902059:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967296256/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767176.902071:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767176.902075:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767176.902077:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767176.902081:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3798 00000100:00080000:0.0:1775767176.902086:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767176.902090:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767176.902096:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767176.902147:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967296384/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767176.902158:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767176.902162:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767176.902164:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767176.902169:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3798 00000100:00080000:0.0:1775767176.902174:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767176.902179:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767176.902182:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967296256:192.168.202.146@tcp:38:kworker.0 00000008:00100000:0.0:1775767176.902188:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767176.902228:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967296512/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767176.902260:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967296384:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767176.902291:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967296512:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767177.797819:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767177.797888:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767178.821851:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767178.821918:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767179.845819:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767179.845908:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775767179.973983:0:2366:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:1.0:1775767179.974021:0:2366:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775767179.974046:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775767179.974094:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775767179.974104:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775767179.974126:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:2.0:1775767179.974138:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767176/real 1775767179] req@ffff8f9970a75c00 x1862022967296256/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767179.974168:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967296256/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767179.974180:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767179.974185:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767179.974190:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967296256:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767179.974199:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767176/real 1775767179] req@ffff8f9970a77480 x1862022967296384/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767179.974212:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967296384/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767179.974220:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767179.974224:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767179.974227:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967296384:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767179.974277:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767176/real 1775767179] req@ffff8f9970a76a00 x1862022967296128/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767179.974293:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967296128/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767179.974301:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767179.974304:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767179.974308:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967296128:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767179.974333:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767176/real 1775767179] req@ffff8f9970a77100 x1862022967296512/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767179.974346:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967296512/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767179.974354:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767179.974357:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767179.974361:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967296512:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767180.869822:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767180.869885:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767181.893789:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767181.893905:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767182.021851:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767182.021862:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767182.021867:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767182.021879:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3803 00000100:00080000:0.0:1775767182.021891:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767182.021898:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767182.021934:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967296640/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767182.021968:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767182.021972:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767182.021974:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767182.021979:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3803 00000100:00080000:0.0:1775767182.021983:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767182.021988:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767182.021998:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767182.022013:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967296768/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767182.022034:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767182.022037:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1775767182.022039:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967296640:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775767182.022040:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767182.022069:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3803 00000100:00080000:0.0:1775767182.022075:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767182.022081:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767182.022087:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767182.022102:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967296896/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767182.022114:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767182.022118:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767182.022120:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767182.022123:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3803 00000100:00080000:0.0:1775767182.022138:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767182.022145:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:2.0:1775767182.022153:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:0.0:1775767182.022156:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767182.022172:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967297024/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767182.022204:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767182/real 1775767182] req@ffff8f9970a75500 x1862022967296640/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767182.022224:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967296640/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767182.022282:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767182.022288:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767182.022295:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967296640:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767182.022335:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967296768:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775767182.022363:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767182.022376:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767182/real 1775767182] req@ffff8f9970a76300 x1862022967296768/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767182.022397:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967296768/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767182.022406:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767182.022410:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767182.022413:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967296768:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767182.022420:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967296896:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767182.022443:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767182.022454:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767182/real 1775767182] req@ffff8f9970a75180 x1862022967296896/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767182.022468:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967296896/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767182.022478:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767182.022482:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767182.022485:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967296896:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767182.022511:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967297024:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767182.022533:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767182.022544:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767182/real 1775767182] req@ffff8f9970a77100 x1862022967297024/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767182.022556:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967297024/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767182.022565:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767182.022568:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767182.022571:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967297024:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767182.917820:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767182.917911:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767183.941899:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767183.941912:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767184.965839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767184.965888:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767185.989852:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767185.989941:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767187.013825:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767187.013865:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767187.141842:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767187.141850:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767187.141853:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767187.141862:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3808 00000100:00080000:1.0:1775767187.141873:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767187.141880:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767187.141921:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967297152/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767187.141945:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767187.141954:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767187.141956:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767187.141959:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3808 00000100:00080000:1.0:1775767187.141962:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767187.141966:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767187.141974:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767187.141986:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967297280/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767187.141991:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967297152:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775767187.142012:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767187.142050:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767187.142051:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767187.142055:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3808 00000100:00080000:1.0:1775767187.142065:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767187.142068:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:2.0:1775767187.142074:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:1.0:1775767187.142075:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767187.142089:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967297408/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767187.142097:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767187.142099:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767187.142100:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767187.142103:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3808 00000100:00080000:1.0:1775767187.142105:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767187.142107:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767187.142110:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767187.142117:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967297536/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767187.142135:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767187/real 1775767187] req@ffff8f994f2e6d80 x1862022967297152/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767187.142156:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967297152/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767187.142168:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767187.142172:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767187.142178:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967297152:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767187.142225:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967297280:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775767187.142260:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767187.142274:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767187/real 1775767187] req@ffff8f994f2e4a80 x1862022967297280/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767187.142289:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967297280/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767187.142298:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767187.142302:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767187.142306:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967297280:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767187.142313:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967297408:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767187.142335:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767187.142347:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767187/real 1775767187] req@ffff8f994f2e7800 x1862022967297408/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767187.142361:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967297408/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767187.142370:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767187.142373:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767187.142377:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967297408:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767187.142384:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967297536:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767187.142415:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767187.142427:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767187/real 1775767187] req@ffff8f994f2e5180 x1862022967297536/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767187.142459:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967297536/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767187.142468:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767187.142473:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767187.142476:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967297536:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767188.037837:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767188.037945:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767189.061807:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767189.061865:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767190.085865:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767190.085935:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767191.109901:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767191.109920:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767192.133818:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767192.133901:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767192.261840:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767192.261848:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767192.261852:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767192.261860:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3813 00000100:00080000:2.0:1775767192.261870:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767192.261877:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767192.261907:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1880 x1862022967297664/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767192.261934:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767192.261937:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767192.261939:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767192.261953:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3813 00000100:00080000:2.0:1775767192.261958:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775767192.261981:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967297664:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775767192.261998:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767192.262007:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767192.262022:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2a00 x1862022967297792/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767192.262031:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767192.262033:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767192.262035:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767192.262039:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3813 00000100:00080000:2.0:1775767192.262042:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767192.262046:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767192.262050:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775767192.262052:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767192.262062:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1c00 x1862022967297920/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767192.262070:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767192.262073:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767192.262074:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767192.262076:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3813 00000100:00080000:2.0:1775767192.262080:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767192.262082:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767192.262086:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767192.262095:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767192/real 1775767192] req@ffff8f99703d1880 x1862022967297664/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767192.262097:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3b80 x1862022967298048/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767192.262129:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1880 x1862022967297664/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767192.262138:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767192.262142:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767192.262145:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967297664:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767192.262163:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967297792:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775767192.262180:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767192.262189:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767192/real 1775767192] req@ffff8f99703d2a00 x1862022967297792/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767192.262197:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2a00 x1862022967297792/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767192.262202:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767192.262204:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767192.262206:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967297792:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767192.262212:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967297920:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767192.262230:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767192.262243:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767192/real 1775767192] req@ffff8f99703d1c00 x1862022967297920/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767192.262254:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1c00 x1862022967297920/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767192.262289:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767192.262293:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767192.262296:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967297920:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767192.262304:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967298048:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767192.262327:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767192.262339:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767192/real 1775767192] req@ffff8f99703d3b80 x1862022967298048/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767192.262353:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3b80 x1862022967298048/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767192.262362:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767192.262367:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767192.262371:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967298048:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767193.157857:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767193.157966:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767194.181813:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767194.181911:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767195.205866:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767195.205972:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767196.229874:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767196.230006:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767197.253828:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767197.253928:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767197.381892:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767197.381919:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767197.381924:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767197.381936:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3818 00000100:00080000:2.0:1775767197.381949:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767197.381956:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767197.382005:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3b80 x1862022967298176/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767197.382031:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767197.382034:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767197.382036:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767197.382039:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3818 00000100:00080000:2.0:1775767197.382041:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767197.382044:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767197.382053:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767197.382063:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1c00 x1862022967298304/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767197.382070:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767197.382073:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767197.382075:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767197.382078:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3818 00000100:00100000:1.0:1775767197.382093:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967298176:192.168.202.146@tcp:250:kworker.0 00000800:00000100:1.0:1775767197.382176:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775767197.382204:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767197.382218:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767197.382234:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767197/real 1775767197] req@ffff8f99703d3b80 x1862022967298176/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1775767197.382234:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767197.382263:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3b80 x1862022967298176/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767197.382280:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2a00 x1862022967298432/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767197.382298:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767197.382303:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767197.382306:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767197.382311:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3818 00000100:00080000:2.0:1775767197.382315:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767197.382319:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767197.382325:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767197.382338:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1880 x1862022967298560/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767197.382385:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767197.382391:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767197.382395:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967298176:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767197.382404:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967298304:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775767197.382458:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767197.382483:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767197/real 1775767197] req@ffff8f99703d1c00 x1862022967298304/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767197.382495:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1c00 x1862022967298304/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767197.382501:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767197.382503:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767197.382505:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967298304:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767197.382524:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967298432:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767197.382535:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767197.382541:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767197/real 1775767197] req@ffff8f99703d2a00 x1862022967298432/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767197.382549:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2a00 x1862022967298432/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767197.382554:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767197.382556:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767197.382558:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967298432:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767197.382562:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967298560:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767197.382571:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767197.382579:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767197/real 1775767197] req@ffff8f99703d1880 x1862022967298560/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767197.382590:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1880 x1862022967298560/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767197.382598:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767197.382601:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767197.382605:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967298560:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767198.277933:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767198.277934:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767199.301837:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767199.301871:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767200.325824:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767200.325926:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767201.349819:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767201.349885:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767202.373820:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767202.373883:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767202.502009:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767202.502029:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767202.502036:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767202.502052:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3824 00000100:00080000:3.0:1775767202.502071:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767202.502083:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767202.502145:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967298688/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767202.502214:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767202.502220:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767202.502396:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767202.502406:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3824 00000100:00080000:3.0:1775767202.502418:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767202.502425:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767202.502440:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767202.502476:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967298816/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767202.502494:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767202.502499:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767202.502502:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767202.502511:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3824 00000100:00080000:3.0:1775767202.502515:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767202.502519:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767202.502525:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767202.502548:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967298944/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767202.502550:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967298688:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775767202.502589:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767202.502592:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767202.502593:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767202.502598:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3824 00000100:00080000:3.0:1775767202.502602:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767202.502607:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767202.502612:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767202.502670:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967299072/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767202.502680:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767202.502821:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767202/real 1775767202] req@ffff8f9970eded80 x1862022967298688/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767202.502865:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967298688/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767202.502879:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767202.502887:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767202.502976:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967298688:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767202.502991:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967298816:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775767202.503023:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767202.503039:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767202/real 1775767202] req@ffff8f9970edd500 x1862022967298816/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767202.503052:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967298816/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767202.503061:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767202.503065:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767202.503069:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967298816:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767202.503116:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967298944:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767202.503141:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767202.503152:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767202/real 1775767202] req@ffff8f9970ede680 x1862022967298944/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767202.503174:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967298944/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767202.503183:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767202.503187:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767202.503191:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967298944:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767202.503199:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967299072:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767202.503222:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767202.503235:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767202/real 1775767202] req@ffff8f9970edc000 x1862022967299072/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767202.503248:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967299072/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767202.503255:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767202.503257:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767202.503259:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967299072:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767203.397912:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767203.397961:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767204.421839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767204.421924:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767205.445901:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767205.445926:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767206.469827:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767206.469903:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767207.493872:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767207.493932:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767207.621848:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767207.621856:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767207.621890:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767207.621898:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3829 00000100:00080000:2.0:1775767207.621907:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767207.621913:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767207.621944:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3480 x1862022967299200/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767207.621966:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767207.621969:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767207.621970:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767207.621973:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3829 00000100:00080000:2.0:1775767207.621975:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767207.621977:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767207.621985:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767207.621996:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1180 x1862022967299328/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767207.622007:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775767207.622012:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967299200:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775767207.622013:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767207.622017:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767207.622023:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3829 00000100:00080000:2.0:1775767207.622027:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767207.622032:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767207.622039:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767207.622053:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2300 x1862022967299456/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767207.622073:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775767207.622084:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767207.622088:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767207.622090:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767207.622094:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3829 00000100:00080000:2.0:1775767207.622097:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767207.622100:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767207.622105:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767207.622111:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767207/real 1775767207] req@ffff8f99703d3480 x1862022967299200/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767207.622118:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3100 x1862022967299584/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767207.622134:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3480 x1862022967299200/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767207.622143:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767207.622147:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767207.622151:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967299200:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767207.622167:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967299328:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775767207.622181:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767207.622190:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767207/real 1775767207] req@ffff8f99703d1180 x1862022967299328/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767207.622212:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1180 x1862022967299328/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767207.622218:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767207.622223:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767207.622225:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967299328:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767207.622229:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967299456:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767207.622248:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767207.622255:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767207/real 1775767207] req@ffff8f99703d2300 x1862022967299456/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767207.622263:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2300 x1862022967299456/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767207.622268:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767207.622270:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767207.622272:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967299456:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767207.622276:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967299584:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767207.622286:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767207.622291:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767207/real 1775767207] req@ffff8f99703d3100 x1862022967299584/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767207.622298:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3100 x1862022967299584/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767207.622307:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767207.622309:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767207.622311:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967299584:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767208.517889:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767208.517967:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767209.541878:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767209.541963:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767210.565853:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767210.565897:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767211.589928:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767211.590011:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767212.613895:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767212.613902:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767212.741870:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767212.741881:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767212.741886:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767212.741897:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3834 00000100:00080000:0.0:1775767212.741908:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767212.741915:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767212.741949:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967299712/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767212.741972:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767212.741975:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767212.741976:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767212.741979:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3834 00000100:00080000:0.0:1775767212.741982:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767212.741985:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767212.741992:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767212.742001:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967299840/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767212.742009:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767212.742012:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767212.742013:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775767212.742018:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967299712:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775767212.742019:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3834 00000100:00080000:0.0:1775767212.742024:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767212.742027:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767212.742031:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767212.742038:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967299968/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767212.742044:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767212.742047:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767212.742048:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767212.742051:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3834 00000100:00080000:0.0:1775767212.742053:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767212.742055:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767212.742058:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767212.742064:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967300096/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767212.742085:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767212.742126:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767212/real 1775767212] req@ffff8f9970a76a00 x1862022967299712/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767212.742147:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967299712/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767212.742158:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767212.742163:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767212.742167:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967299712:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767212.742190:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967299840:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775767212.742213:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767212.742226:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767212/real 1775767212] req@ffff8f9970a77480 x1862022967299840/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767212.742240:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967299840/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767212.742288:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767212.742291:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767212.742293:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967299840:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767212.742298:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967299968:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767212.742316:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767212.742324:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767212/real 1775767212] req@ffff8f9970a75c00 x1862022967299968/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767212.742332:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967299968/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767212.742338:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767212.742340:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767212.742342:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967299968:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767212.742345:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967300096:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767212.742355:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767212.742361:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767212/real 1775767212] req@ffff8f9970a74380 x1862022967300096/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767212.742368:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967300096/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767212.742373:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767212.742374:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767212.742377:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967300096:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767213.637862:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767213.637922:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767214.661843:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767214.661901:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767215.685848:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767215.685913:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767216.709850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767216.709910:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767217.733838:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767217.733902:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767217.861872:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767217.861883:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767217.861888:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767217.861899:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3839 00000100:00080000:0.0:1775767217.861910:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767217.861916:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767217.861954:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967300224/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767217.861980:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767217.861982:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767217.861984:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767217.861988:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3839 00000100:00080000:0.0:1775767217.861991:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767217.861994:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767217.862001:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767217.862011:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967300352/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767217.862018:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767217.862020:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767217.862021:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767217.862024:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3839 00000100:00080000:0.0:1775767217.862032:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767217.862035:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767217.862038:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767217.862046:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967300224:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767217.862047:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967300480/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767217.862061:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767217.862063:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767217.862064:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767217.862066:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3839 00000100:00080000:0.0:1775767217.862068:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767217.862069:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767217.862073:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767217.862085:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967300608/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767217.862122:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767217.862183:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767217/real 1775767217] req@ffff8f9970a77800 x1862022967300224/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767217.862205:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967300224/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767217.862216:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767217.862220:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767217.862224:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967300224:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767217.862232:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967300352:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775767217.862256:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767217.862283:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767217/real 1775767217] req@ffff8f9970a76d80 x1862022967300352/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767217.862297:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967300352/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767217.862306:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767217.862309:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767217.862313:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967300352:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767217.862340:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967300480:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767217.862360:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767217.862371:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767217/real 1775767217] req@ffff8f9970a74700 x1862022967300480/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767217.862382:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967300480/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767217.862390:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767217.862394:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767217.862397:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967300480:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767217.862403:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967300608:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767217.862421:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767217.862441:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767217/real 1775767217] req@ffff8f9970a74380 x1862022967300608/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767217.862454:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967300608/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767217.862462:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767217.862466:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767217.862470:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967300608:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767218.757832:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767218.757941:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767219.781907:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767219.781910:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775767220.805797:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767220.805946:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767221.829867:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767221.829933:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767222.853845:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767222.853921:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767222.981901:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767222.981910:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767222.981942:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767222.981951:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3844 00000100:00080000:2.0:1775767222.981961:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767222.981966:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767222.982000:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0e00 x1862022967300736/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767222.982030:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767222.982033:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767222.982034:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767222.982038:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3844 00000100:00080000:2.0:1775767222.982040:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767222.982042:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767222.982050:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767222.982057:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2d80 x1862022967300864/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767222.982064:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767222.982066:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767222.982067:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767222.982070:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3844 00000100:00080000:2.0:1775767222.982071:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767222.982074:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767222.982077:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767222.982083:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1500 x1862022967300992/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767222.982108:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767222.982111:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767222.982113:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767222.982120:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3844 00000100:00080000:2.0:1775767222.982123:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767222.982125:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767222.982128:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967300736:192.168.202.146@tcp:250:kworker.0 00000008:00100000:2.0:1775767222.982131:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767222.982149:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0700 x1862022967301120/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767222.982217:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767222.982265:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767222/real 1775767222] req@ffff8f99703d0e00 x1862022967300736/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767222.982282:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0e00 x1862022967300736/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767222.982291:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767222.982295:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767222.982298:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967300736:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767222.982304:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967300864:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775767222.982319:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767222.982327:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767222/real 1775767222] req@ffff8f99703d2d80 x1862022967300864/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767222.982335:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2d80 x1862022967300864/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767222.982341:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767222.982343:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767222.982344:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967300864:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767222.982367:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967300992:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767222.982378:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767222.982384:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767222/real 1775767222] req@ffff8f99703d1500 x1862022967300992/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767222.982397:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1500 x1862022967300992/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767222.982402:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767222.982404:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767222.982405:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967300992:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767222.982409:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967301120:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767222.982419:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767222.982425:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767222/real 1775767222] req@ffff8f99703d0700 x1862022967301120/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767222.982432:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0700 x1862022967301120/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767222.982438:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767222.982439:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767222.982441:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967301120:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767223.877802:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767223.877931:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767224.901838:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767224.901887:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767225.925814:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767225.925880:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767226.949913:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767226.949934:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767227.973844:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767227.973904:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767228.101887:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767228.101896:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767228.101901:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767228.101911:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3849 00000100:00080000:2.0:1775767228.101922:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767228.101929:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767228.101964:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3800 x1862022967301248/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767228.101986:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767228.101989:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767228.102018:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775767228.102019:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967301248:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775767228.102025:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3849 00000100:00080000:2.0:1775767228.102036:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767228.102041:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767228.102051:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767228.102064:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0380 x1862022967301376/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767228.102079:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767228.102083:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767228.102085:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767228.102090:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3849 00000100:00080000:2.0:1775767228.102093:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767228.102096:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767228.102103:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767228.102118:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0000 x1862022967301504/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767228.102125:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767228.102128:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767228.102131:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1775767228.102140:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775767228.102146:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3849 00000100:00080000:2.0:1775767228.102150:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767228.102155:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767228.102163:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767228.102186:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2680 x1862022967301632/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767228.102216:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767228/real 1775767228] req@ffff8f99703d3800 x1862022967301248/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767228.102332:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3800 x1862022967301248/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767228.102344:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767228.102349:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767228.102354:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967301248:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767228.102390:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967301376:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775767228.102437:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767228.102458:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767228/real 1775767228] req@ffff8f99703d0380 x1862022967301376/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767228.102470:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0380 x1862022967301376/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767228.102476:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767228.102478:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767228.102481:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967301376:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767228.102488:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967301504:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767228.102508:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767228.102522:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767228/real 1775767228] req@ffff8f99703d0000 x1862022967301504/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767228.102532:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0000 x1862022967301504/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767228.102558:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767228.102561:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767228.102563:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967301504:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767228.102567:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967301632:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767228.102580:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767228.102587:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767228/real 1775767228] req@ffff8f99703d2680 x1862022967301632/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767228.102594:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2680 x1862022967301632/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767228.102599:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767228.102601:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767228.102603:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967301632:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767228.997838:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767228.997925:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767230.021809:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767230.021892:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767231.045842:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767231.045892:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767232.069832:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767232.069889:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767233.093850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767233.093959:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767233.221840:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767233.221851:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767233.221855:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767233.221866:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3854 00000100:00080000:2.0:1775767233.221878:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767233.221884:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767233.221925:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2680 x1862022967301760/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767233.221956:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767233.221960:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767233.221962:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767233.221967:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3854 00000100:00080000:2.0:1775767233.221971:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767233.221975:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767233.221985:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767233.221999:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0000 x1862022967301888/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767233.222011:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775767233.222013:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967301760:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775767233.222043:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767233.222046:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767233.222051:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3854 00000100:00080000:2.0:1775767233.222056:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767233.222060:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767233.222066:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775767233.222079:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767233.222081:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0380 x1862022967302016/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767233.222093:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767233.222098:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767233.222100:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767233.222104:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3854 00000100:00080000:2.0:1775767233.222109:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767233.222114:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767233.222118:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767233.222131:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767233/real 1775767233] req@ffff8f99703d2680 x1862022967301760/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767233.222133:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3800 x1862022967302144/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767233.222173:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2680 x1862022967301760/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767233.222192:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767233.222196:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767233.222200:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967301760:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767233.222222:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967301888:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775767233.222242:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767233.222253:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767233/real 1775767233] req@ffff8f99703d0000 x1862022967301888/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767233.222266:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0000 x1862022967301888/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767233.222275:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767233.222278:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767233.222282:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967301888:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767233.222288:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967302016:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767233.222304:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767233.222313:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767233/real 1775767233] req@ffff8f99703d0380 x1862022967302016/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767233.222324:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0380 x1862022967302016/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767233.222332:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767233.222336:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767233.222339:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967302016:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767233.222365:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967302144:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767233.222383:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767233.222393:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767233/real 1775767233] req@ffff8f99703d3800 x1862022967302144/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767233.222405:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3800 x1862022967302144/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767233.222413:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767233.222417:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767233.222420:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967302144:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767234.117921:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767234.117955:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767235.141867:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767235.141918:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767236.165855:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767236.165944:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767237.189860:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767237.189935:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767238.213873:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767238.213910:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767238.341853:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767238.341864:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767238.341942:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767238.341954:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3859 00000100:00080000:2.0:1775767238.341968:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767238.341975:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767238.342025:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0700 x1862022967302272/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767238.342055:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767238.342058:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767238.342059:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767238.342063:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3859 00000100:00080000:2.0:1775767238.342065:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767238.342068:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767238.342078:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767238.342089:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1500 x1862022967302400/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767238.342098:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767238.342101:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767238.342102:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767238.342110:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3859 00000100:00080000:2.0:1775767238.342115:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775767238.342119:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967302272:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775767238.342120:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767238.342126:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767238.342138:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2d80 x1862022967302528/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767238.342166:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767238.342169:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767238.342171:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767238.342175:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3859 00000100:00080000:2.0:1775767238.342178:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767238.342181:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767238.342185:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767238.342193:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0e00 x1862022967302656/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767238.342208:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767238.342257:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767238/real 1775767238] req@ffff8f99703d0700 x1862022967302272/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767238.342274:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0700 x1862022967302272/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767238.342283:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767238.342287:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767238.342291:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967302272:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767238.342343:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967302400:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775767238.342369:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767238.342383:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767238/real 1775767238] req@ffff8f99703d1500 x1862022967302400/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767238.342395:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1500 x1862022967302400/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767238.342405:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767238.342409:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767238.342412:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967302400:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767238.342419:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967302528:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767238.342439:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767238.342450:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767238/real 1775767238] req@ffff8f99703d2d80 x1862022967302528/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767238.342464:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2d80 x1862022967302528/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767238.342473:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767238.342476:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767238.342480:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967302528:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767238.342486:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967302656:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767238.342505:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767238.342535:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767238/real 1775767238] req@ffff8f99703d0e00 x1862022967302656/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767238.342549:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0e00 x1862022967302656/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767238.342557:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767238.342561:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767238.342564:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967302656:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767239.237870:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767239.237919:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767240.261817:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767240.261878:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767241.285875:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767241.285885:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767242.309805:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767242.309860:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767243.333939:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767243.333976:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00080000:0.0:1775767243.461858:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767243.461872:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767243.461879:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767243.461894:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3865 00000100:00080000:0.0:1775767243.461911:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767243.461919:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767243.461971:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967302784/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767243.462007:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767243.462012:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767243.462014:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767243.462020:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3865 00000100:00080000:0.0:1775767243.462025:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767243.462029:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767243.462041:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767243.462057:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967302912/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767243.462069:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767243.462074:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767243.462076:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767243.462082:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3865 00000100:00080000:0.0:1775767243.462086:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767243.462093:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767243.462102:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767243.462104:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967302784:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767243.462120:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967303040/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767243.462136:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767243.462140:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767243.462142:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767243.462148:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3865 00000100:00080000:0.0:1775767243.462154:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767243.462159:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767243.462164:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767243.462187:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967303168/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767243.462233:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967302912:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767243.462280:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967303040:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767243.462307:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967303168:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767244.357853:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767244.357926:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767245.381823:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767245.381919:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767246.405844:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767246.405908:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775767246.534096:0:2368:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:1.0:1775767246.534144:0:2368:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775767246.534161:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775767246.534207:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775767246.534218:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775767246.534227:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:2.0:1775767246.534257:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767243/real 1775767246] req@ffff8f9970a76a00 x1862022967303040/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767246.534287:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967303040/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767246.534299:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767246.534305:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767246.534310:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967303040:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767246.534393:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767243/real 1775767246] req@ffff8f9970a77100 x1862022967303168/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767246.534408:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967303168/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767246.534417:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767246.534431:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767246.534461:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967303168:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767246.534469:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767243/real 1775767246] req@ffff8f9970a75c00 x1862022967302784/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767246.534484:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967302784/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767246.534492:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767246.534496:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767246.534499:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967302784:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767246.534507:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767243/real 1775767246] req@ffff8f9970a77480 x1862022967302912/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767246.534520:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967302912/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767246.534528:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767246.534532:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767246.534536:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967302912:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767247.429811:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767247.429890:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767248.453929:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767248.453956:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767248.581970:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767248.581988:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767248.581993:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767248.582003:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3870 00000100:00080000:2.0:1775767248.582020:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767248.582029:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767248.582078:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3100 x1862022967303296/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767248.582134:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767248.582140:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767248.582143:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767248.582150:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3870 00000100:00080000:2.0:1775767248.582158:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767248.582165:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767248.582194:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767248.582200:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967303296:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767248.582211:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2300 x1862022967303424/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767248.582223:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767248.582226:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767248.582228:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767248.582232:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3870 00000100:00080000:2.0:1775767248.582236:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1775767248.582307:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767248.582409:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767248/real 1775767248] req@ffff8f99703d3100 x1862022967303296/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767248.582449:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767248.582472:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767248.582477:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3100 x1862022967303296/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767248.582495:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767248.582501:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767248.582506:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967303296:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767248.582527:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1180 x1862022967303552/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767248.582553:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967303424:192.168.202.146@tcp:38:kworker.0 00000100:00080000:2.0:1775767248.582555:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767248.582564:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767248.582568:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767248.582576:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3870 00000100:00080000:2.0:1775767248.582592:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1775767248.582604:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775767248.582604:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767248.582614:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767248.582634:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3480 x1862022967303680/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767248.582645:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767248/real 1775767248] req@ffff8f99703d2300 x1862022967303424/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767248.582687:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2300 x1862022967303424/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767248.582698:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767248.582703:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767248.582710:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967303424:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767248.582733:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967303552:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767248.582804:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767248.582818:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767248/real 1775767248] req@ffff8f99703d1180 x1862022967303552/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767248.582830:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1180 x1862022967303552/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767248.582839:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767248.582842:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767248.582845:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967303552:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767248.582851:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967303680:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767248.582873:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767248.582884:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767248/real 1775767248] req@ffff8f99703d3480 x1862022967303680/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767248.582895:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3480 x1862022967303680/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767248.582904:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767248.582908:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767248.582912:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967303680:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767249.477808:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767249.477915:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767250.501863:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767250.501957:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767251.525822:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767251.525889:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767252.549831:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767252.549919:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767253.573896:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767253.573955:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767253.701908:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767253.701922:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767253.701928:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767253.701942:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3875 00000100:00080000:0.0:1775767253.701957:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767253.701966:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767253.702018:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967303808/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767253.702066:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767253.702082:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767253.702084:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767253.702090:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3875 00000100:00080000:0.0:1775767253.702095:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767253.702100:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767253.702112:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767253.702129:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967303936/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767253.702152:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967303808:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775767253.702154:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767253.702164:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767253.702166:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767253.702170:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3875 00000100:00080000:0.0:1775767253.702175:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767253.702180:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767253.702187:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767253.702203:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967304064/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767253.702214:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767253.702218:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767253.702220:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767253.702225:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3875 00000100:00080000:0.0:1775767253.702230:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767253.702234:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767253.702240:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775767253.702243:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767253.702260:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967304192/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767253.702304:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767253/real 1775767253] req@ffff8f9970a75180 x1862022967303808/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767253.702336:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967303808/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767253.702346:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767253.702349:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767253.702352:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967303808:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767253.702393:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967303936:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775767253.702419:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767253.702434:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767253/real 1775767253] req@ffff8f9970a76300 x1862022967303936/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767253.702448:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967303936/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767253.702457:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767253.702461:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767253.702465:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967303936:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767253.702472:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967304064:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767253.702494:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767253.702502:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767253/real 1775767253] req@ffff8f9970a75500 x1862022967304064/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767253.702511:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967304064/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767253.702521:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767253.702523:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767253.702526:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967304064:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767253.702529:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967304192:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767253.702541:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767253.702547:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767253/real 1775767253] req@ffff8f9970a77480 x1862022967304192/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767253.702554:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967304192/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767253.702559:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767253.702561:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767253.702562:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967304192:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767254.597877:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767254.597953:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767255.621848:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767255.621959:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775767256.645878:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767256.645925:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767257.669820:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767257.669981:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767258.693863:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767258.693984:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767258.821832:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767258.821843:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767258.821848:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767258.821859:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3880 00000100:00080000:0.0:1775767258.821870:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767258.821878:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767258.821936:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967304320/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767258.821972:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767258.821977:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767258.821980:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767258.822001:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3880 00000100:00080000:0.0:1775767258.822006:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767258.822010:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767258.822021:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767258.822038:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967304448/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767258.822042:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967304320:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775767258.822113:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1775767258.822118:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775767258.822121:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767258.822124:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767258.822129:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3880 00000100:00080000:0.0:1775767258.822135:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767258.822141:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767258.822147:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767258.822158:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767258/real 1775767258] req@ffff8f9970a75c00 x1862022967304320/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767258.822165:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967304576/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767258.822180:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967304320/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767258.822182:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767258.822191:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767258.822195:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767258.822203:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3880 00000100:00080000:1.0:1775767258.822205:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767258.822210:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767258.822212:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00080000:0.0:1775767258.822220:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767258.822221:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967304320:192.168.202.146@tcp:250:kworker.0 00000008:00100000:0.0:1775767258.822231:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767258.822244:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967304448:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767258.822248:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967304704/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767258.822264:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767258.822275:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767258/real 1775767258] req@ffff8f9970a77100 x1862022967304448/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767258.822284:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967304448/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767258.822306:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767258.822309:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767258.822311:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967304448:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767258.822315:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967304576:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767258.822372:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767258.822380:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767258/real 1775767258] req@ffff8f9970a76a00 x1862022967304576/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767258.822387:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967304576/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767258.822393:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767258.822396:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767258.822398:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967304576:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767258.822408:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967304704:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767258.822419:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767258.822424:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767258/real 1775767258] req@ffff8f9970a74380 x1862022967304704/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767258.822431:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967304704/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767258.822436:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767258.822438:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767258.822440:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967304704:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767259.717836:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767259.717906:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767260.741823:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767260.741896:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767261.765816:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767261.765890:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767262.789889:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767262.789895:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775767263.813835:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767263.813901:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767263.941828:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767263.941843:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767263.941847:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767263.941855:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3885 00000100:00080000:0.0:1775767263.941865:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767263.941872:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767263.941908:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967304832/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767263.941929:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767263.941932:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767263.941933:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767263.941936:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3885 00000100:00080000:0.0:1775767263.941939:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767263.941942:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767263.941950:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767263.941963:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967304832:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767263.941964:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967304960/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767263.941980:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767263.941983:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767263.941985:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767263.941989:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3885 00000100:00080000:0.0:1775767263.941992:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767263.941994:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767263.941998:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767263.942008:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967305088/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767263.942015:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1775767263.942025:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775767263.942054:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767263.942056:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775767263.942060:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767263/real 1775767263] req@ffff8f9970a74700 x1862022967304832/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767263.942063:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3885 00000100:00080000:0.0:1775767263.942072:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767263.942077:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767263.942081:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967304832/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1775767263.942085:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1775767263.942091:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767263.942095:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767263.942099:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967304832:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767263.942105:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967305216/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767263.942142:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967304960:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775767263.942165:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767263.942178:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767263/real 1775767263] req@ffff8f9970a76d80 x1862022967304960/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767263.942192:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967304960/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767263.942201:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767263.942205:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767263.942208:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967304960:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767263.942215:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967305088:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767263.942230:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767263.942238:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767263/real 1775767263] req@ffff8f9970a77800 x1862022967305088/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767263.942246:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967305088/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767263.942251:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767263.942254:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767263.942256:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967305088:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767263.942259:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967305216:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767263.942269:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767263.942291:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767263/real 1775767263] req@ffff8f9970a74380 x1862022967305216/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767263.942313:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967305216/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767263.942319:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767263.942321:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767263.942324:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967305216:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767264.837824:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767264.837886:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767265.861857:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767265.861916:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767266.885826:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767266.885907:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767267.909867:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767267.909960:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767268.933829:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767268.933938:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767269.061913:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767269.061930:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767269.061939:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767269.061954:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3890 00000100:00080000:0.0:1775767269.061972:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767269.061982:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767269.062044:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967305344/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767269.062088:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767269.062106:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767269.062109:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767269.062115:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3890 00000100:00080000:0.0:1775767269.062121:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767269.062126:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767269.062140:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767269.062158:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967305472/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767269.062168:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767269.062172:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767269.062174:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767269.062178:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3890 00000100:00080000:0.0:1775767269.062182:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767269.062186:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767269.062206:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767269.062218:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967305600/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767269.062244:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767269.062248:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767269.062250:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767269.062254:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3890 00000100:00080000:0.0:1775767269.062258:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767269.062262:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767269.062274:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767269.062288:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967305728/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767269.062544:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967305344:192.168.202.146@tcp:250:kworker.0 00000800:00000100:2.0:1775767269.062640:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767269.062723:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767269/real 1775767269] req@ffff8f9970a76a00 x1862022967305344/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767269.062787:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967305344/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767269.062797:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767269.062801:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767269.062804:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967305344:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767269.062810:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967305472:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775767269.062831:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767269.062841:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767269/real 1775767269] req@ffff8f9970a77100 x1862022967305472/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767269.062850:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967305472/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767269.062855:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767269.062858:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767269.062860:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967305472:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767269.062985:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967305600:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767269.063047:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767269.063069:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767269/real 1775767269] req@ffff8f9970a75c00 x1862022967305600/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767269.063083:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967305600/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767269.063091:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767269.063094:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767269.063097:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967305600:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767269.063102:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967305728:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767269.063115:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767269.063121:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767269/real 1775767269] req@ffff8f9970a77480 x1862022967305728/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767269.063129:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967305728/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767269.063134:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767269.063136:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767269.063138:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967305728:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775767269.957957:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767269.957957:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775767270.981847:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767270.981919:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767272.005841:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767272.005927:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767273.029845:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767273.029915:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767274.053848:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767274.053924:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767274.181845:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767274.181853:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767274.181857:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767274.181867:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3895 00000100:00080000:0.0:1775767274.181876:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767274.181882:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767274.181921:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967305856/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767274.181952:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767274.181956:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767274.181958:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767274.181962:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3895 00000100:00080000:0.0:1775767274.181966:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767274.181970:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767274.181978:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767274.181980:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967305856:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767274.181992:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967305984/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767274.182001:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767274.182004:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767274.182005:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767274.182008:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3895 00000100:00080000:0.0:1775767274.182010:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767274.182013:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767274.182016:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767274.182023:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967306112/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767274.182037:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767274.182071:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767274/real 1775767274] req@ffff8f9970a75500 x1862022967305856/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767274.182094:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775767274.182099:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967305856/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767274.182100:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767274.182105:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767274.182114:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3895 00000100:00080000:2.0:1775767274.182114:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767274.182119:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767274.182122:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00080000:0.0:1775767274.182126:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767274.182129:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967305856:192.168.202.146@tcp:250:kworker.0 00000008:00100000:0.0:1775767274.182132:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767274.182146:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967306240/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767274.182160:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967305984:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775767274.182195:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767274.182208:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767274/real 1775767274] req@ffff8f9970a76300 x1862022967305984/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767274.182219:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967305984/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767274.182227:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767274.182231:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767274.182234:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967305984:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767274.182239:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967306112:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767274.182255:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767274.182271:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767274/real 1775767274] req@ffff8f9970a75180 x1862022967306112/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767274.182308:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967306112/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767274.182318:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767274.182321:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767274.182325:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967306112:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767274.182331:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967306240:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767274.182352:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767274.182363:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767274/real 1775767274] req@ffff8f9970a77480 x1862022967306240/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767274.182373:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967306240/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767274.182379:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767274.182381:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767274.182383:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967306240:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767275.077856:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767275.077911:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767276.101844:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767276.101927:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767277.125919:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767277.125919:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775767278.149829:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767278.149879:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767279.173848:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767279.173936:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767279.301824:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767279.301833:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767279.301838:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767279.301850:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3900 00000100:00080000:1.0:1775767279.301864:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767279.301871:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767279.301914:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967306368/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767279.302054:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767279.302058:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767279.302061:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767279.302069:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3900 00000100:00080000:1.0:1775767279.302079:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767279.302091:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767279.302099:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967306368:192.168.202.146@tcp:250:kworker.0 00000008:00100000:1.0:1775767279.302101:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767279.302143:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967306496/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767279.302152:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767279.302155:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767279.302156:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767279.302159:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3900 00000100:00080000:1.0:1775767279.302163:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767279.302166:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767279.302170:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775767279.302170:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767279.302181:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967306624/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767279.302189:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767279.302191:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767279.302192:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767279.302194:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3900 00000100:00100000:2.0:1775767279.302215:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767279/real 1775767279] req@ffff8f994f2e5180 x1862022967306368/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767279.302220:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767279.302223:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767279.302226:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767279.302236:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967306752/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767279.302260:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967306368/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767279.302275:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767279.302279:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767279.302285:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967306368:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767279.302305:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967306496:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775767279.302322:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767279.302330:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767279/real 1775767279] req@ffff8f994f2e7800 x1862022967306496/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767279.302338:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967306496/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767279.302344:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767279.302346:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767279.302348:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967306496:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767279.302351:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967306624:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767279.302362:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767279.302367:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767279/real 1775767279] req@ffff8f994f2e4a80 x1862022967306624/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767279.302375:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967306624/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767279.302396:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767279.302398:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767279.302401:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967306624:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767279.302405:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967306752:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767279.302416:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767279.302423:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767279/real 1775767279] req@ffff8f994f2e6d80 x1862022967306752/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767279.302430:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967306752/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767279.302435:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767279.302437:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767279.302438:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967306752:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767280.197828:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767280.197898:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767281.221828:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767281.221854:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767282.245849:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767282.245904:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767283.269863:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767283.269917:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767284.293886:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767284.293906:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767284.421855:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767284.421865:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767284.421870:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767284.421881:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3906 00000100:00080000:0.0:1775767284.421892:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767284.421899:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767284.421936:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967306880/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767284.421963:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767284.421967:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767284.421969:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767284.421975:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3906 00000100:00080000:0.0:1775767284.421980:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775767284.421988:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967306880:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775767284.422000:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767284.422010:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767284.422027:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967307008/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767284.422039:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767284.422043:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767284.422045:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1775767284.422049:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775767284.422054:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3906 00000100:00080000:0.0:1775767284.422060:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767284.422065:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767284.422072:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767284.422101:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967307136/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767284.422112:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767284.422115:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767284.422117:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767284.422122:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3906 00000100:00100000:2.0:1775767284.422127:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767284/real 1775767284] req@ffff8f9970a75c00 x1862022967306880/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767284.422128:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767284.422137:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767284.422147:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767284.422160:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967306880/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767284.422162:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967307264/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767284.422210:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767284.422214:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767284.422217:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967306880:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767284.422235:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967307008:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775767284.422258:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767284.422271:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767284/real 1775767284] req@ffff8f9970a77100 x1862022967307008/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767284.422286:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967307008/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767284.422295:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767284.422299:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767284.422302:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967307008:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767284.422321:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967307136:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767284.422349:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767284.422361:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767284/real 1775767284] req@ffff8f9970a76a00 x1862022967307136/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767284.422373:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967307136/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767284.422379:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767284.422381:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767284.422384:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967307136:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767284.422402:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967307264:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767284.422417:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767284.422424:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767284/real 1775767284] req@ffff8f9970a74380 x1862022967307264/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767284.422432:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967307264/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767284.422437:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767284.422440:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767284.422442:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967307264:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775767285.317897:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767285.317957:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775767286.341817:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767286.341923:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767287.365890:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767287.365959:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767288.389921:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767288.389933:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767289.413906:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775767289.413950:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767289.541858:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767289.541869:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767289.541874:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767289.541885:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3911 00000100:00080000:1.0:1775767289.541897:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767289.541904:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767289.541940:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967307392/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767289.541969:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767289.541973:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767289.541975:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767289.541980:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3911 00000100:00080000:1.0:1775767289.541984:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767289.541990:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767289.541999:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967307392:192.168.202.146@tcp:250:kworker.0 00000008:00100000:1.0:1775767289.542000:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767289.542019:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967307520/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767289.542031:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767289.542049:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767289.542051:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767289.542056:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3911 00000100:00080000:1.0:1775767289.542060:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767289.542065:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767289.542071:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775767289.542079:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767289.542085:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967307648/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767289.542097:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767289.542101:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767289.542103:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767289.542108:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3911 00000100:00080000:1.0:1775767289.542113:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775767289.542118:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767289/real 1775767289] req@ffff8f994f2e6680 x1862022967307392/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767289.542119:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767289.542128:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767289.542138:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967307392/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767289.542144:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967307776/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767289.542148:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767289.542156:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767289.542162:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967307392:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767289.542217:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967307520:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775767289.542234:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767289.542242:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767289/real 1775767289] req@ffff8f994f2e4000 x1862022967307520/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767289.542251:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967307520/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767289.542256:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767289.542259:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767289.542261:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967307520:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767289.542265:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967307648:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767289.542275:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767289.542281:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767289/real 1775767289] req@ffff8f994f2e5f80 x1862022967307648/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767289.542288:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967307648/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767289.542293:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767289.542295:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767289.542297:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967307648:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767289.542300:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967307776:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767289.542309:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767289.542315:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767289/real 1775767289] req@ffff8f994f2e6a00 x1862022967307776/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767289.542335:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967307776/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767289.542340:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767289.542342:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767289.542344:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967307776:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767290.437815:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767290.437918:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767291.461883:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767291.461912:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767292.485836:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767292.485896:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767293.509845:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767293.509895:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767294.533813:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767294.533870:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767294.661837:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767294.661845:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767294.661849:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767294.661859:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3916 00000100:00080000:1.0:1775767294.661871:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767294.661878:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767294.661916:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967307904/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767294.661941:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767294.661944:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767294.661945:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767294.661948:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3916 00000100:00080000:1.0:1775767294.661951:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767294.661954:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767294.661963:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767294.661976:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967308032/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767294.662002:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967307904:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775767294.662003:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767294.662010:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767294.662013:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767294.662019:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3916 00000100:00080000:1.0:1775767294.662025:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767294.662030:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767294.662036:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767294.662048:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967308160/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767294.662056:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767294.662058:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767294.662059:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767294.662062:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3916 00000800:00000100:2.0:1775767294.662066:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775767294.662067:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767294.662072:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767294.662075:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767294.662084:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967308288/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767294.662111:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767294/real 1775767294] req@ffff8f994f2e4700 x1862022967307904/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767294.662128:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967307904/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767294.662137:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767294.662140:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767294.662143:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967307904:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767294.662161:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967308032:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775767294.662176:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767294.662184:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767294/real 1775767294] req@ffff8f994f2e6300 x1862022967308032/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767294.662218:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967308032/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767294.662224:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767294.662226:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767294.662228:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967308032:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767294.662232:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967308160:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767294.662245:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767294.662252:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767294/real 1775767294] req@ffff8f994f2e4e00 x1862022967308160/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767294.662259:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967308160/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767294.662273:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767294.662275:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767294.662278:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967308160:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767294.662281:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967308288:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767294.662293:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767294.662299:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767294/real 1775767294] req@ffff8f994f2e7b80 x1862022967308288/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767294.662306:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967308288/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767294.662325:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767294.662328:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767294.662330:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967308288:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767295.557817:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767295.557883:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767296.581853:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767296.581898:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767297.605836:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767297.605898:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767298.629872:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767298.629894:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767299.653817:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767299.653888:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767299.781823:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767299.781838:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767299.781843:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767299.781851:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3921 00000100:00080000:1.0:1775767299.781862:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767299.781868:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767299.781901:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967308416/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767299.781925:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767299.781928:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767299.781929:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767299.781932:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3921 00000100:00080000:1.0:1775767299.781935:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767299.781959:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767299.781969:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767299.781986:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967308544/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767299.781997:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967308416:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775767299.782003:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767299.782008:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767299.782010:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767299.782014:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3921 00000100:00080000:1.0:1775767299.782018:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767299.782022:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767299.782027:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767299.782039:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967308672/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767299.782054:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767299.782056:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767299.782058:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1775767299.782063:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775767299.782095:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3921 00000100:00100000:0.0:1775767299.782096:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767299/real 1775767299] req@ffff8f994f2e7b80 x1862022967308416/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767299.782101:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767299.782108:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767299.782113:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767299.782121:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967308416/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767299.782126:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967308800/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767299.782138:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767299.782146:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767299.782153:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967308416:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767299.782179:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967308544:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775767299.782204:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767299.782215:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767299/real 1775767299] req@ffff8f994f2e4e00 x1862022967308544/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767299.782224:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967308544/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767299.782230:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767299.782232:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767299.782235:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967308544:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767299.782239:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967308672:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767299.782254:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767299.782267:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767299/real 1775767299] req@ffff8f994f2e6300 x1862022967308672/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767299.782275:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967308672/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767299.782281:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767299.782283:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767299.782285:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967308672:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767299.782290:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967308800:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767299.782301:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767299.782309:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767299/real 1775767299] req@ffff8f994f2e4700 x1862022967308800/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767299.782322:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967308800/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767299.782330:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767299.782334:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767299.782337:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967308800:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767300.677823:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767300.677862:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767301.701814:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767301.701876:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767302.725900:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767302.725989:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767303.749809:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767303.749935:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767304.773853:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767304.773967:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767304.901914:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767304.901923:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767304.901927:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767304.901936:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3926 00000100:00080000:2.0:1775767304.901947:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767304.901953:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767304.901984:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1880 x1862022967308928/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767304.902008:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767304.902012:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767304.902014:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767304.902019:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3926 00000100:00080000:2.0:1775767304.902056:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767304.902061:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767304.902065:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967308928:192.168.202.146@tcp:250:kworker.0 00000008:00100000:2.0:1775767304.902074:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767304.902089:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2a00 x1862022967309056/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767304.902098:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767304.902100:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767304.902101:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767304.902104:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3926 00000100:00080000:2.0:1775767304.902107:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767304.902111:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767304.902116:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767304.902127:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1c00 x1862022967309184/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767304.902136:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767304.902143:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:0.0:1775767304.902144:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775767304.902146:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767304.902152:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3926 00000100:00080000:2.0:1775767304.902157:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767304.902171:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767304.902181:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767304.902191:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3b80 x1862022967309312/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767304.902200:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767304/real 1775767304] req@ffff8f99703d1880 x1862022967308928/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767304.902243:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1880 x1862022967308928/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767304.902255:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767304.902260:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767304.902263:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967308928:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767304.902283:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967309056:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775767304.902308:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767304.902331:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767304/real 1775767304] req@ffff8f99703d2a00 x1862022967309056/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767304.902345:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2a00 x1862022967309056/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767304.902354:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767304.902358:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767304.902362:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967309056:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767304.902368:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967309184:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767304.902390:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767304.902401:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767304/real 1775767304] req@ffff8f99703d1c00 x1862022967309184/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767304.902409:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1c00 x1862022967309184/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767304.902445:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767304.902450:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767304.902454:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967309184:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767304.902461:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967309312:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767304.902483:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767304.902495:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767304/real 1775767304] req@ffff8f99703d3b80 x1862022967309312/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767304.902504:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3b80 x1862022967309312/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767304.902510:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767304.902512:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767304.902514:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967309312:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767305.797939:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767305.798153:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767306.821905:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767306.821959:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767307.845833:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767307.845913:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767308.869832:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767308.869943:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767309.893872:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767309.893929:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767310.021888:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767310.021896:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767310.021901:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767310.021911:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3931 00000100:00080000:2.0:1775767310.021922:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767310.021928:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767310.021964:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3b80 x1862022967309440/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767310.021987:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767310.021990:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767310.021992:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767310.021996:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3931 00000100:00080000:2.0:1775767310.021998:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767310.022001:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767310.022009:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767310.022019:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1c00 x1862022967309568/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767310.022027:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767310.022033:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1775767310.022035:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967309440:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775767310.022068:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767310.022073:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3931 00000100:00080000:2.0:1775767310.022075:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767310.022078:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767310.022082:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767310.022094:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2a00 x1862022967309696/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767310.022102:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767310.022104:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767310.022105:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767310.022108:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3931 00000100:00080000:2.0:1775767310.022110:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767310.022113:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767310.022115:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767310.022121:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1880 x1862022967309824/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767310.022193:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967309568:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767310.022237:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967309696:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767310.022268:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967309824:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767310.917884:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767310.917988:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767311.941868:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767311.941946:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767312.965871:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767312.965887:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775767313.093995:0:2367:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:1.0:1775767313.094031:0:2367:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775767313.094045:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775767313.094085:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775767313.094094:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775767313.094102:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:0.0:1775767313.094138:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767310/real 1775767313] req@ffff8f99703d1c00 x1862022967309568/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767313.094175:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1c00 x1862022967309568/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767313.094221:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767313.094228:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767313.094238:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967309568:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767313.094247:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767310/real 1775767313] req@ffff8f99703d2a00 x1862022967309696/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767313.094261:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2a00 x1862022967309696/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767313.094272:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767313.094275:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767313.094280:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967309696:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767313.094287:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767310/real 1775767313] req@ffff8f99703d1880 x1862022967309824/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767313.094298:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1880 x1862022967309824/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767313.094306:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767313.094309:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767313.094313:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967309824:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767313.094319:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767310/real 1775767313] req@ffff8f99703d3b80 x1862022967309440/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767313.094333:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3b80 x1862022967309440/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767313.094342:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767313.094345:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767313.094349:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967309440:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767313.989849:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767313.989897:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767315.013853:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767315.013915:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767315.141917:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767315.141925:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767315.141929:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767315.141938:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3936 00000100:00080000:1.0:1775767315.141948:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767315.141954:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767315.141998:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967309952/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767315.142027:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767315.142030:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767315.142032:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767315.142037:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3936 00000100:00080000:1.0:1775767315.142043:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767315.142049:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767315.142061:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767315.142070:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967309952:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767315.142078:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967310080/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767315.142090:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767315.142094:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767315.142096:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767315.142107:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3936 00000100:00080000:1.0:1775767315.142128:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767315.142133:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:0.0:1775767315.142139:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:1.0:1775767315.142140:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767315.142156:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967310208/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767315.142172:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767315.142176:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767315.142179:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767315.142185:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3936 00000100:00080000:1.0:1775767315.142189:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767315.142192:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767315.142197:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767315.142202:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767315/real 1775767315] req@ffff8f994f2e6a00 x1862022967309952/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767315.142213:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967310336/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767315.142225:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967309952/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767315.142250:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767315.142255:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767315.142261:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967309952:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767315.142286:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967310080:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775767315.142312:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767315.142326:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767315/real 1775767315] req@ffff8f994f2e5f80 x1862022967310080/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767315.142340:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967310080/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767315.142348:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767315.142352:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767315.142355:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967310080:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767315.142363:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967310208:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767315.142384:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767315.142395:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767315/real 1775767315] req@ffff8f994f2e4000 x1862022967310208/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767315.142409:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967310208/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767315.142418:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767315.142421:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767315.142425:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967310208:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767315.142431:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967310336:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767315.142451:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767315.142463:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767315/real 1775767315] req@ffff8f994f2e6680 x1862022967310336/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767315.142476:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967310336/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767315.142485:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767315.142488:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767315.142491:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967310336:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767316.037821:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767316.037883:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767317.061842:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767317.061903:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767318.085867:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767318.085928:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767319.109851:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767319.109940:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767320.133901:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767320.133917:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767320.261870:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767320.261879:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767320.261883:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767320.261892:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3941 00000100:00080000:1.0:1775767320.261902:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767320.261907:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767320.261946:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967310464/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767320.261968:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767320.261970:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767320.262002:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767320.262006:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3941 00000100:00080000:1.0:1775767320.262008:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767320.262011:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767320.262019:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767320.262028:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967310592/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767320.262037:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767320.262040:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767320.262041:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767320.262044:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3941 00000100:00080000:1.0:1775767320.262046:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767320.262049:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767320.262051:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767320.262060:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967310720/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767320.262070:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767320.262072:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767320.262074:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767320.262081:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3941 00000100:00100000:0.0:1775767320.262084:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967310464:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775767320.262085:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767320.262093:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767320.262103:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767320.262112:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967310848/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767320.262197:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767320.262247:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767320/real 1775767320] req@ffff8f994f2e6d80 x1862022967310464/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767320.262314:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967310464/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767320.262327:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767320.262333:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767320.262338:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967310464:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767320.262348:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967310592:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775767320.262377:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767320.262409:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767320/real 1775767320] req@ffff8f994f2e4a80 x1862022967310592/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767320.262423:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967310592/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767320.262432:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767320.262436:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767320.262440:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967310592:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767320.262474:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967310720:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767320.262499:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767320.262512:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767320/real 1775767320] req@ffff8f994f2e7800 x1862022967310720/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767320.262527:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967310720/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767320.262544:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767320.262548:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767320.262551:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967310720:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767320.262558:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967310848:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767320.262581:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767320.262594:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767320/real 1775767320] req@ffff8f994f2e5180 x1862022967310848/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767320.262607:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967310848/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767320.262615:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767320.262618:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767320.262620:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967310848:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767321.157850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767321.157946:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767322.181811:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767322.181926:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767323.205820:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767323.205874:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767324.229921:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767324.229976:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775767325.253818:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767325.253870:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767325.381886:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767325.381897:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767325.381902:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767325.381912:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3946 00000100:00080000:2.0:1775767325.381924:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767325.381946:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767325.381987:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3480 x1862022967310976/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767325.382017:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767325.382021:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767325.382023:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767325.382028:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3946 00000100:00080000:2.0:1775767325.382032:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767325.382036:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767325.382046:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767325.382061:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1180 x1862022967311104/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767325.382064:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967310976:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775767325.382081:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767325.382084:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767325.382087:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1775767325.382128:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775767325.382136:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3946 00000100:00080000:2.0:1775767325.382141:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767325.382146:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767325.382152:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767325.382167:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2300 x1862022967311232/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767325.382178:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767325.382184:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1775767325.382185:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767325/real 1775767325] req@ffff8f99703d3480 x1862022967310976/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767325.382187:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767325.382199:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3946 00000100:00080000:2.0:1775767325.382208:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767325.382214:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767325.382217:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3480 x1862022967310976/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1775767325.382221:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1775767325.382232:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767325.382235:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767325.382241:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967310976:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767325.382242:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3100 x1862022967311360/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767325.382265:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967311104:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775767325.382286:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767325.382295:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767325/real 1775767325] req@ffff8f99703d1180 x1862022967311104/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767325.382304:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1180 x1862022967311104/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767325.382309:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767325.382312:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767325.382314:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967311104:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767325.382318:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967311232:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767325.382329:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767325.382335:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767325/real 1775767325] req@ffff8f99703d2300 x1862022967311232/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767325.382342:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2300 x1862022967311232/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767325.382347:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767325.382349:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767325.382351:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967311232:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767325.382354:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967311360:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767325.382364:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767325.382372:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767325/real 1775767325] req@ffff8f99703d3100 x1862022967311360/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767325.382384:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3100 x1862022967311360/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767325.382392:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767325.382395:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767325.382399:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967311360:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767326.277861:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767326.277932:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767327.301861:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767327.301889:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767328.325842:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767328.325918:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767329.349819:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767329.349884:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767330.373861:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767330.373910:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767330.501868:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767330.501876:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767330.501880:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767330.501888:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3952 00000100:00080000:0.0:1775767330.501898:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767330.501903:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767330.501937:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967311488/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767330.501964:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767330.501968:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767330.501970:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767330.501974:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3952 00000100:00080000:0.0:1775767330.501979:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767330.501983:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767330.501991:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967311488:192.168.202.146@tcp:250:kworker.0 00000008:00100000:0.0:1775767330.501993:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767330.502011:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967311616/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767330.502038:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767330.502041:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767330.502042:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767330.502046:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3952 00000800:00000100:2.0:1775767330.502051:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775767330.502053:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767330.502059:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767330.502064:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767330.502075:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967311744/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767330.502082:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767330.502084:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767330.502085:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767330.502087:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3952 00000100:00080000:0.0:1775767330.502089:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767330.502092:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767330.502096:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767330.502096:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767330/real 1775767330] req@ffff8f9970a77800 x1862022967311488/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767330.502113:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967311872/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767330.502166:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967311488/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767330.502174:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767330.502177:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767330.502180:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967311488:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767330.502198:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967311616:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775767330.502217:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767330.502225:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767330/real 1775767330] req@ffff8f9970a76d80 x1862022967311616/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767330.502234:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967311616/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767330.502239:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767330.502241:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767330.502243:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967311616:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767330.502247:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967311744:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767330.502257:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767330.502264:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767330/real 1775767330] req@ffff8f9970a74700 x1862022967311744/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767330.502277:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967311744/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767330.502308:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767330.502312:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767330.502316:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967311744:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767330.502322:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967311872:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767330.502344:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767330.502355:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767330/real 1775767330] req@ffff8f9970a74380 x1862022967311872/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767330.502368:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967311872/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767330.502377:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767330.502381:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767330.502384:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967311872:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767331.397837:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767331.397889:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767332.421846:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767332.421912:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767333.445851:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767333.445905:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767334.469886:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767334.469923:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767335.493817:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767335.493885:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767335.622692:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767335.622700:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767335.622705:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767335.622717:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3957 00000100:00080000:3.0:1775767335.622729:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767335.622738:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767335.622858:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967312000/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767335.622893:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767335.622898:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767335.622900:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767335.622907:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3957 00000100:00080000:3.0:1775767335.622912:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767335.622917:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767335.622928:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767335.622933:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967312000:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767335.622943:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967312128/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767335.622953:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767335.622956:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767335.622958:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767335.622982:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3957 00000100:00080000:3.0:1775767335.622987:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767335.622991:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767335.622997:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767335.623010:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967312256/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767335.623020:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775767335.623025:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767335.623029:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767335.623031:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767335.623037:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3957 00000100:00080000:3.0:1775767335.623041:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767335.623047:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767335.623052:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767335.623057:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767335/real 1775767335] req@ffff8f9970edf800 x1862022967312000/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767335.623068:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967312384/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767335.623074:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967312000/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767335.623093:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767335.623097:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767335.623100:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967312000:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767335.623118:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967312128:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775767335.623134:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767335.623141:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767335/real 1775767335] req@ffff8f9970edf480 x1862022967312128/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767335.623149:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967312128/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767335.623155:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767335.623156:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767335.623158:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967312128:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767335.623170:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967312256:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767335.623185:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767335.623191:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767335/real 1775767335] req@ffff8f9970edd880 x1862022967312256/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767335.623198:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967312256/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767335.623203:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767335.623205:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767335.623207:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967312256:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767335.623211:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967312384:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767335.623223:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767335.623233:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767335/real 1775767335] req@ffff8f9970ede300 x1862022967312384/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767335.623252:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967312384/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767335.623260:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767335.623263:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767335.623266:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967312384:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767336.517834:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767336.517892:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767337.541796:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767337.541880:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767338.565851:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767338.565900:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767339.589821:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767339.589869:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767340.613841:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767340.613932:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767340.741838:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767340.741847:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767340.741851:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767340.741860:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3962 00000100:00080000:3.0:1775767340.741872:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767340.741881:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767340.741921:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967312512/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767340.741950:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775767340.741994:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967312512:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775767340.742004:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767340.742006:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767340.742012:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3962 00000100:00080000:3.0:1775767340.742017:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767340.742021:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767340.742032:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767340.742045:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967312640/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767340.742054:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1775767340.742056:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775767340.742060:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767340.742062:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767340.742066:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3962 00000100:00080000:3.0:1775767340.742070:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767340.742073:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767340.742077:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767340.742085:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967312768/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767340.742091:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767340.742093:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767340.742094:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767340.742097:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3962 00000100:00080000:3.0:1775767340.742100:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767340.742103:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767340.742107:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767340/real 1775767340] req@ffff8f9970edfb80 x1862022967312512/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767340.742122:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967312512/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1775767340.742126:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1775767340.742135:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767340.742141:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767340.742141:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967312896/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767340.742149:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967312512:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767340.742176:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967312640:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775767340.742193:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767340.742201:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767340/real 1775767340] req@ffff8f9970edc380 x1862022967312640/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767340.742209:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967312640/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767340.742215:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767340.742217:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767340.742219:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967312640:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767340.742223:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967312768:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767340.742232:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767340.742240:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767340/real 1775767340] req@ffff8f9970edca80 x1862022967312768/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767340.742252:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967312768/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767340.742260:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767340.742263:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767340.742266:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967312768:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767340.742272:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967312896:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767340.742292:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767340.742302:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767340/real 1775767340] req@ffff8f9970edea00 x1862022967312896/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767340.742311:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967312896/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767340.742320:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767340.742322:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767340.742324:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967312896:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775767341.637941:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767341.638021:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775767342.661835:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767342.661934:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767343.685907:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767343.686002:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767344.709866:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767344.710007:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767345.733858:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767345.733922:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767345.861867:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767345.861878:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767345.861883:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767345.861895:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3967 00000100:00080000:3.0:1775767345.861906:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767345.861913:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767345.861946:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddf80 x1862022967313024/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767345.861969:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767345.861972:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767345.861973:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767345.861977:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3967 00000100:00080000:3.0:1775767345.861979:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767345.861981:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767345.861988:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767345.861997:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd180 x1862022967313152/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767345.862004:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767345.862006:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767345.862007:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767345.862010:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3967 00000100:00100000:1.0:1775767345.862017:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967313024:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775767345.862048:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767345.862051:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767345.862055:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767345.862065:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967313280/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767345.862072:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767345.862074:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767345.862076:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767345.862078:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3967 00000100:00080000:3.0:1775767345.862081:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1775767345.862085:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775767345.862088:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767345.862092:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767345.862099:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967313408/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767345.862147:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767345/real 1775767345] req@ffff8f9970eddf80 x1862022967313024/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767345.862168:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddf80 x1862022967313024/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767345.862180:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767345.862185:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767345.862189:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967313024:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767345.862198:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967313152:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775767345.862220:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767345.862233:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767345/real 1775767345] req@ffff8f9970edd180 x1862022967313152/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767345.862247:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd180 x1862022967313152/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767345.862256:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767345.862260:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767345.862264:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967313152:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767345.862289:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967313280:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767345.862311:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767345.862322:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767345/real 1775767345] req@ffff8f9970edea00 x1862022967313280/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767345.862335:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967313280/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767345.862344:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767345.862348:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767345.862351:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967313280:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767345.862358:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967313408:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767345.862376:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767345.862386:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767345/real 1775767345] req@ffff8f9970edca80 x1862022967313408/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767345.862399:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967313408/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767345.862408:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767345.862411:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767345.862435:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967313408:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767346.757807:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767346.757944:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767347.781883:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767347.781943:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767348.805876:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767348.805898:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767349.829865:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767349.829944:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767350.853825:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767350.853941:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767350.981911:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767350.981922:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767350.981927:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767350.981938:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3972 00000100:00080000:0.0:1775767350.981951:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767350.981958:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767350.981994:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967313536/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767350.982016:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767350.982018:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767350.982020:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767350.982023:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3972 00000100:00080000:0.0:1775767350.982026:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767350.982028:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767350.982036:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767350.982044:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967313664/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767350.982053:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967313536:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775767350.982055:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767350.982062:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767350.982063:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767350.982066:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3972 00000100:00080000:0.0:1775767350.982068:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767350.982070:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767350.982074:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767350.982081:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967313792/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767350.982090:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767350.982095:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767350.982096:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767350.982100:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3972 00000100:00080000:0.0:1775767350.982105:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767350.982109:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:1.0:1775767350.982114:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:0.0:1775767350.982117:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767350.982132:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967313920/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767350.982166:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767350/real 1775767350] req@ffff8f9970a76a00 x1862022967313536/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767350.982188:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967313536/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767350.982199:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767350.982203:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767350.982206:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967313536:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767350.982227:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967313664:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775767350.982245:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767350.982254:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767350/real 1775767350] req@ffff8f9970a77100 x1862022967313664/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767350.982263:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967313664/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767350.982269:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767350.982271:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767350.982299:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967313664:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767350.982303:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967313792:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767350.982317:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767350.982325:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767350/real 1775767350] req@ffff8f9970a75c00 x1862022967313792/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767350.982333:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967313792/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767350.982338:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767350.982340:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767350.982342:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967313792:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767350.982346:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967313920:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767350.982356:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767350.982362:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767350/real 1775767350] req@ffff8f9970a77480 x1862022967313920/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767350.982369:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967313920/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767350.982374:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767350.982376:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767350.982377:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967313920:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767351.877902:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767351.877960:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767352.901853:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767352.901960:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767353.925873:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767353.925968:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767354.949863:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767354.949954:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767355.973922:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767355.973954:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00080000:0.0:1775767356.101872:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767356.101887:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767356.101894:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767356.101907:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3977 00000100:00080000:0.0:1775767356.101922:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767356.101930:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767356.101980:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967314048/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767356.102017:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767356.102022:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767356.102024:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767356.102029:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3977 00000100:00080000:0.0:1775767356.102033:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767356.102038:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767356.102051:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767356.102067:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967314176/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767356.102078:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967314048:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775767356.102085:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767356.102089:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767356.102092:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767356.102097:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3977 00000100:00080000:0.0:1775767356.102101:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767356.102106:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767356.102113:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767356.102126:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967314304/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767356.102138:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767356.102142:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767356.102144:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767356.102148:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3977 00000100:00080000:0.0:1775767356.102162:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767356.102167:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:1.0:1775767356.102173:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:0.0:1775767356.102176:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767356.102193:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967314432/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767356.102230:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767356/real 1775767356] req@ffff8f9970a75180 x1862022967314048/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767356.102255:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967314048/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767356.102266:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767356.102271:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767356.102275:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967314048:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767356.102301:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967314176:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775767356.102330:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767356.102339:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767356/real 1775767356] req@ffff8f9970a76300 x1862022967314176/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767356.102348:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967314176/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767356.102353:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767356.102356:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767356.102358:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967314176:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767356.102362:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967314304:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767356.102376:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767356.102382:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767356/real 1775767356] req@ffff8f9970a75500 x1862022967314304/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767356.102390:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967314304/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767356.102395:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767356.102397:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767356.102400:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967314304:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767356.102406:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967314432:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767356.102439:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767356.102450:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767356/real 1775767356] req@ffff8f9970a77480 x1862022967314432/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767356.102465:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967314432/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767356.102475:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767356.102477:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767356.102480:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967314432:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767356.997876:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767356.998012:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767358.021829:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767358.021903:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767359.045899:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767359.045928:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767360.069820:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767360.069917:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767361.093895:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767361.093941:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00080000:3.0:1775767361.221853:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767361.221863:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767361.221868:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767361.221877:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3982 00000100:00080000:3.0:1775767361.221886:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767361.221892:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767361.221927:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967314560/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767361.221953:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767361.221992:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767361.221994:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767361.222000:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3982 00000100:00100000:1.0:1775767361.222001:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967314560:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775767361.222003:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767361.222009:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767361.222018:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767361.222033:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967314688/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767361.222055:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767361.222059:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767361.222061:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767361.222065:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3982 00000800:00000100:1.0:1775767361.222070:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775767361.222071:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767361.222078:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767361.222085:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767361.222098:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967314816/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767361.222109:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767361.222112:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767361.222114:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767361.222117:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3982 00000100:00080000:3.0:1775767361.222122:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775767361.222127:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767361/real 1775767361] req@ffff8f9970edc380 x1862022967314560/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767361.222128:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767361.222145:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967314560/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1775767361.222154:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1775767361.222157:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767361.222163:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767361.222170:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967314944/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767361.222171:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967314560:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767361.222204:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967314688:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775767361.222228:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767361.222238:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767361/real 1775767361] req@ffff8f9970edfb80 x1862022967314688/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767361.222247:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967314688/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767361.222252:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767361.222255:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767361.222257:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967314688:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767361.222261:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967314816:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767361.222273:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767361.222279:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767361/real 1775767361] req@ffff8f9970ede300 x1862022967314816/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767361.222297:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967314816/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767361.222305:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767361.222308:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767361.222312:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967314816:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767361.222329:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967314944:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767361.222349:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767361.222358:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767361/real 1775767361] req@ffff8f9970edd880 x1862022967314944/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767361.222381:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967314944/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767361.222386:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767361.222389:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767361.222391:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967314944:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767362.117876:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767362.117882:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767363.141929:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767363.141961:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775767364.165848:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767364.165911:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767365.189912:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767365.189922:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767366.213832:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767366.213883:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767366.341854:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767366.341862:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767366.341867:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767366.341877:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3987 00000100:00080000:3.0:1775767366.341887:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767366.341892:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767366.341926:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967315072/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767366.341953:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767366.341957:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767366.341959:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767366.341964:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3987 00000100:00080000:3.0:1775767366.341968:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767366.341973:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767366.341984:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767366.342001:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967315200/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767366.342011:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967315072:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775767366.342023:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767366.342027:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767366.342029:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767366.342036:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3987 00000100:00080000:3.0:1775767366.342073:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767366.342077:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767366.342082:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767366.342094:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967315328/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767366.342102:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767366.342107:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:1.0:1775767366.342108:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775767366.342110:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767366.342115:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3987 00000100:00080000:3.0:1775767366.342118:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767366.342121:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767366.342125:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767366.342133:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967315456/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767366.342173:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767366/real 1775767366] req@ffff8f9970edf480 x1862022967315072/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767366.342192:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967315072/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767366.342209:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767366.342215:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767366.342219:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967315072:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767366.342246:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967315200:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775767366.342265:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767366.342275:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767366/real 1775767366] req@ffff8f9970edf800 x1862022967315200/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767366.342283:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967315200/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767366.342289:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767366.342292:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767366.342294:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967315200:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767366.342298:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967315328:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767366.342310:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767366.342328:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767366/real 1775767366] req@ffff8f9970edc000 x1862022967315328/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767366.342338:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967315328/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767366.342349:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767366.342352:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767366.342354:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967315328:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767366.342359:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967315456:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767366.342375:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767366.342386:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767366/real 1775767366] req@ffff8f9970ede680 x1862022967315456/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767366.342398:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967315456/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767366.342407:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767366.342410:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767366.342414:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967315456:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767367.237830:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767367.237897:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767368.261847:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767368.261906:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767369.285837:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767369.285931:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767370.309868:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767370.309898:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767371.333844:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767371.333896:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767371.461837:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767371.461845:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767371.461850:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767371.461860:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3993 00000100:00080000:3.0:1775767371.461870:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767371.461875:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767371.461908:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967315584/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767371.461930:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775767371.461956:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967315584:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775767371.461977:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767371.461995:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767371.462000:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3993 00000100:00080000:3.0:1775767371.462004:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767371.462008:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767371.462017:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767371.462031:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967315712/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767371.462034:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775767371.462068:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767371.462071:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767371.462073:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767371.462078:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3993 00000100:00080000:3.0:1775767371.462081:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767371.462085:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767371.462090:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767371.462093:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767371/real 1775767371] req@ffff8f9970edd500 x1862022967315584/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767371.462102:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967315840/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767371.462109:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767371.462111:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767371.462112:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767371.462114:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3993 00000100:00080000:3.0:1775767371.462116:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767371.462118:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767371.462131:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767371.462139:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967315968/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767371.462220:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967315584/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767371.462234:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767371.462240:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767371.462244:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967315584:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767371.462306:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967315712:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775767371.462355:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767371.462375:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767371/real 1775767371] req@ffff8f9970eded80 x1862022967315712/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767371.462390:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967315712/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767371.462400:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767371.462404:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767371.462409:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967315712:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767371.462419:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967315840:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767371.462445:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767371.462465:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767371/real 1775767371] req@ffff8f9970edce00 x1862022967315840/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767371.462474:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967315840/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767371.462509:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767371.462518:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767371.462523:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967315840:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767371.462541:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967315968:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767371.462567:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767371.462580:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767371/real 1775767371] req@ffff8f9970edc700 x1862022967315968/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767371.462588:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967315968/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767371.462594:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767371.462596:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767371.462598:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967315968:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767372.357847:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767372.358014:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767373.381864:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767373.381907:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767374.405812:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767374.405934:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767375.429815:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767375.429897:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767376.453818:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767376.453921:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767376.581834:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767376.581845:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767376.581849:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767376.581857:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 3998 00000100:00080000:3.0:1775767376.581869:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767376.581876:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767376.581911:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf100 x1862022967316096/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767376.581939:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767376.581942:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767376.581943:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767376.581947:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3998 00000100:00080000:3.0:1775767376.581950:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767376.581953:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767376.581961:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767376.581972:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddc00 x1862022967316224/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767376.581980:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767376.581982:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767376.581985:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775767376.581987:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967316096:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775767376.581989:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3998 00000100:00080000:3.0:1775767376.582021:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767376.582026:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767376.582031:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767376.582047:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967316352/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767376.582058:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767376.582062:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767376.582064:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767376.582069:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 3998 00000100:00080000:3.0:1775767376.582072:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767376.582075:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767376.582080:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767376.582091:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967316480/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767376.582095:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967316224:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767376.582128:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967316352:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767376.582146:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967316480:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775767377.477875:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767377.477913:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775767378.501899:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767378.501951:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767379.525886:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767379.525976:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000400:00000100:1.0:1775767379.653974:0:2369:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:1.0:1775767379.654015:0:2369:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775767379.654034:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775767379.654075:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775767379.654085:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775767379.654094:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:2.0:1775767379.654190:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767376/real 1775767379] req@ffff8f9970edce00 x1862022967316480/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767379.654255:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967316480/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767379.654274:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767379.654283:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767379.654291:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967316480:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767379.654304:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767376/real 1775767379] req@ffff8f9970eddc00 x1862022967316224/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767379.654322:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddc00 x1862022967316224/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767379.654334:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767379.654338:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767379.654343:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967316224:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767379.654354:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767376/real 1775767379] req@ffff8f9970edc700 x1862022967316352/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767379.654370:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967316352/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767379.654381:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767379.654453:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767379.654458:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967316352:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767379.654470:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767376/real 1775767379] req@ffff8f9970edf100 x1862022967316096/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767379.654488:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf100 x1862022967316096/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767379.654498:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767379.654502:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767379.654507:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967316096:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767380.549868:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767380.549921:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767381.573854:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767381.573920:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767381.701915:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767381.701923:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767381.701927:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767381.701936:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4003 00000100:00080000:0.0:1775767381.701946:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767381.701951:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767381.701988:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967316608/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767381.702011:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767381.702014:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767381.702015:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767381.702019:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4003 00000100:00080000:0.0:1775767381.702021:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767381.702023:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767381.702031:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767381.702039:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967316736/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767381.702047:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767381.702049:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767381.702051:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767381.702054:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4003 00000100:00080000:0.0:1775767381.702059:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767381.702061:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767381.702064:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767381.702070:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967316864/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767381.702077:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767381.702081:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767381.702082:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767381.702084:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4003 00000100:00080000:0.0:1775767381.702088:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767381.702092:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767381.702099:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767381.702100:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967316608:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767381.702108:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967316992/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767381.702253:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767381.702328:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767381/real 1775767381] req@ffff8f9970a75c00 x1862022967316608/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767381.702353:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967316608/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767381.702367:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767381.702374:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767381.702378:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967316608:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767381.702387:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967316736:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775767381.702416:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767381.702499:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767381/real 1775767381] req@ffff8f9970a77100 x1862022967316736/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767381.702512:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967316736/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767381.702521:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767381.702525:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767381.702529:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967316736:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767381.702565:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967316864:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767381.702597:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767381.702610:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767381/real 1775767381] req@ffff8f9970a76a00 x1862022967316864/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767381.702623:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967316864/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767381.702631:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767381.702635:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767381.702638:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967316864:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767381.702645:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967316992:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767381.702676:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767381.702689:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767381/real 1775767381] req@ffff8f9970a74380 x1862022967316992/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767381.702701:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967316992/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767381.702729:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767381.702734:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767381.702781:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967316992:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767382.597940:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767382.598019:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767383.621850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767383.621940:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767384.645922:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767384.645969:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775767385.669864:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767385.669966:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767386.693884:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767386.693945:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767386.821874:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767386.821888:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767386.821894:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767386.821908:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4008 00000100:00080000:3.0:1775767386.821922:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767386.821932:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767386.821982:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967317120/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767386.822015:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775767386.822063:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967317120:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775767386.822081:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767386.822084:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767386.822094:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4008 00000100:00080000:3.0:1775767386.822102:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767386.822110:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767386.822143:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767386.822163:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967317248/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767386.822176:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767386.822180:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767386.822182:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767386.822186:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4008 00000100:00080000:3.0:1775767386.822191:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767386.822199:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:2.0:1775767386.822201:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:3.0:1775767386.822213:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767386.822224:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967317376/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767386.822235:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767386.822240:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767386.822253:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767386.822256:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4008 00000100:00080000:3.0:1775767386.822262:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767386.822265:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767386.822276:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767386/real 1775767386] req@ffff8f9970eded80 x1862022967317120/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1775767386.822293:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767386.822305:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967317120/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767386.822311:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967317504/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767386.822372:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767386.822382:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767386.822391:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967317120:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767386.822423:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967317248:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775767386.822446:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767386.822455:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767386/real 1775767386] req@ffff8f9970edd500 x1862022967317248/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767386.822464:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967317248/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767386.822510:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767386.822513:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767386.822515:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967317248:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767386.822521:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967317376:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767386.822546:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767386.822554:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767386/real 1775767386] req@ffff8f9970ede680 x1862022967317376/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767386.822562:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967317376/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767386.822567:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767386.822570:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767386.822581:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967317376:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767386.822586:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967317504:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767386.822598:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767386.822605:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767386/real 1775767386] req@ffff8f9970edc000 x1862022967317504/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767386.822613:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967317504/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767386.822618:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767386.822620:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767386.822634:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967317504:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767387.717903:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767387.717950:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767388.741870:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767388.741976:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767389.765869:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767389.765936:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767390.789834:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767390.789927:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767391.813910:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767391.813939:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767391.941851:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767391.941860:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767391.941863:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767391.941872:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4013 00000100:00080000:1.0:1775767391.941883:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767391.941889:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767391.941923:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967317632/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767391.941952:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767391.941956:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767391.941958:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767391.941965:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4013 00000100:00080000:1.0:1775767391.941970:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767391.941976:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767391.941987:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767391.941996:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967317632:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767391.942011:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967317760/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767391.942025:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767391.942028:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767391.942029:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767391.942033:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4013 00000100:00080000:1.0:1775767391.942036:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767391.942039:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767391.942044:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767391.942054:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967317888/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767391.942060:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767391.942064:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767391.942066:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767391.942071:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4013 00000100:00080000:1.0:1775767391.942075:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767391.942091:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767391.942097:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775767391.942103:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767391.942117:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967318016/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767391.942176:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767391/real 1775767391] req@ffff8f994f2e5180 x1862022967317632/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767391.942197:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967317632/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767391.942210:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767391.942216:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767391.942221:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967317632:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767391.942255:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967317760:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775767391.942276:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767391.942287:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767391/real 1775767391] req@ffff8f994f2e7800 x1862022967317760/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767391.942295:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967317760/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767391.942302:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767391.942304:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767391.942332:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967317760:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767391.942338:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967317888:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767391.942352:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767391.942359:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767391/real 1775767391] req@ffff8f994f2e4a80 x1862022967317888/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767391.942367:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967317888/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767391.942373:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767391.942375:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767391.942378:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967317888:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767391.942382:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967318016:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767391.942393:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767391.942399:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767391/real 1775767391] req@ffff8f994f2e6d80 x1862022967318016/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767391.942407:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967318016/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767391.942412:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767391.942414:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767391.942415:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967318016:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767392.837890:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767392.837948:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767393.861814:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767393.861941:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767394.885836:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767394.885905:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767395.909827:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767395.909887:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767396.933824:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767396.933936:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767397.061825:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767397.061832:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767397.061836:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767397.061843:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4018 00000100:00080000:1.0:1775767397.061853:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767397.061858:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767397.061890:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967318144/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767397.061910:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767397.061913:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767397.061914:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767397.061917:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4018 00000100:00080000:1.0:1775767397.061919:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767397.061922:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767397.061931:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767397.061947:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967318272/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767397.061947:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967318144:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775767397.061965:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767397.061968:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767397.061971:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767397.061975:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4018 00000100:00080000:1.0:1775767397.061980:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767397.061984:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767397.061991:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767397.062001:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967318400/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767397.062008:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1775767397.062010:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775767397.062021:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767397.062023:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767397.062027:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4018 00000100:00080000:1.0:1775767397.062043:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767397.062048:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767397.062053:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767397.062062:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967318528/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767397.062070:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767397/real 1775767397] req@ffff8f994f2e6680 x1862022967318144/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767397.062088:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967318144/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767397.062096:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767397.062099:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767397.062103:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967318144:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767397.062121:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967318272:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775767397.062139:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767397.062147:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767397/real 1775767397] req@ffff8f994f2e4000 x1862022967318272/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767397.062155:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967318272/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767397.062161:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767397.062163:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767397.062165:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967318272:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767397.062169:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967318400:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767397.062333:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767397.062357:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767397/real 1775767397] req@ffff8f994f2e5f80 x1862022967318400/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767397.062370:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967318400/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767397.062378:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767397.062381:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767397.062385:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967318400:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767397.062393:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967318528:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767397.062406:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767397.062412:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767397/real 1775767397] req@ffff8f994f2e6a00 x1862022967318528/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767397.062420:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967318528/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767397.062425:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767397.062427:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767397.062429:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967318528:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767397.957849:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767397.957918:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767398.981874:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767398.981879:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767400.005821:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767400.005897:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767401.029856:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767401.029967:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767402.053893:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767402.053972:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767402.182464:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767402.182481:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767402.182487:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767402.182505:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4023 00000100:00080000:3.0:1775767402.182523:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767402.182533:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767402.182591:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967318656/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767402.182624:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767402.182628:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767402.182631:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767402.182636:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4023 00000100:00080000:3.0:1775767402.182641:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767402.182645:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767402.182658:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767402.182676:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967318784/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767402.182690:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967318656:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775767402.182691:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767402.182695:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767402.182697:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767402.182702:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4023 00000800:00000100:2.0:1775767402.182842:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775767402.182854:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767402.182869:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767402.182885:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767402.182917:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767402/real 1775767402] req@ffff8f9970edf800 x1862022967318656/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767402.182928:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967318912/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767402.182958:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775767402.182960:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967318656/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767402.182970:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767402.182979:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767402.183004:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767402.183007:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4023 00000100:00080000:2.0:1775767402.183019:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00080000:3.0:1775767402.183019:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775767402.183040:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967318656:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775767402.183040:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767402.183058:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767402.183074:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967319040/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767402.183085:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967318784:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775767402.183120:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767402.183136:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767402/real 1775767402] req@ffff8f9970edf480 x1862022967318784/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767402.183150:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967318784/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767402.183159:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767402.183163:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767402.183167:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967318784:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767402.183173:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967318912:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767402.183192:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767402.183199:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767402/real 1775767402] req@ffff8f9970edd880 x1862022967318912/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767402.183233:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967318912/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767402.183242:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767402.183245:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767402.183249:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967318912:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767402.183266:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967319040:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767402.183288:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767402.183299:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767402/real 1775767402] req@ffff8f9970ede300 x1862022967319040/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767402.183307:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967319040/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767402.183313:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767402.183316:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767402.183318:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967319040:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767403.077838:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767403.077931:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767404.101885:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767404.101918:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767405.125794:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767405.125930:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767406.150064:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767406.150066:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767407.173896:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767407.173975:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767407.301889:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767407.301899:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767407.301904:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767407.301917:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4028 00000100:00080000:3.0:1775767407.301930:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767407.301954:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767407.301989:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967319168/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767407.302017:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775767407.302062:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967319168:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775767407.302073:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767407.302076:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767407.302092:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4028 00000100:00080000:3.0:1775767407.302097:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767407.302102:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767407.302112:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767407.302131:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967319296/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767407.302134:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775767407.302150:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767407.302154:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767407.302157:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767407.302161:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4028 00000100:00080000:3.0:1775767407.302164:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767407.302168:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767407.302173:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767407.302191:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767407/real 1775767407] req@ffff8f9970edfb80 x1862022967319168/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767407.302217:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967319168/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767407.302231:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767407.302236:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767407.302240:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967319168:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767407.302257:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967319424/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767407.302264:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967319296:192.168.202.146@tcp:38:kworker.0 00000100:00080000:3.0:1775767407.302268:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767407.302271:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767407.302273:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767407.302276:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4028 00000100:00080000:3.0:1775767407.302280:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767407.302284:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:0.0:1775767407.302285:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767407.302298:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767407/real 1775767407] req@ffff8f9970edc380 x1862022967319296/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1775767407.302306:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767407.302311:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967319296/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767407.302316:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967319552/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767407.302320:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767407.302323:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767407.302326:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967319296:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767407.302339:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967319424:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767407.302359:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767407.302371:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767407/real 1775767407] req@ffff8f9970edca80 x1862022967319424/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767407.302380:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967319424/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767407.302385:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767407.302387:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767407.302389:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967319424:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767407.302393:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967319552:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767407.302406:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767407.302412:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767407/real 1775767407] req@ffff8f9970edea00 x1862022967319552/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767407.302423:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967319552/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767407.302431:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767407.302435:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767407.302438:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967319552:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767408.197876:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767408.197913:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767409.221913:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767409.222006:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767410.245850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767410.245934:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767411.269833:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767411.269924:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767412.293918:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767412.293939:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767412.421874:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767412.421909:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767412.421915:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767412.421926:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4034 00000100:00080000:1.0:1775767412.421937:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767412.421943:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767412.421975:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967319680/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767412.421998:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767412.422000:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767412.422002:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767412.422010:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4034 00000100:00080000:1.0:1775767412.422012:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767412.422015:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767412.422023:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767412.422032:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967319808/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767412.422040:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767412.422043:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1775767412.422045:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967319680:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775767412.422046:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767412.422053:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4034 00000100:00080000:1.0:1775767412.422055:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767412.422058:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767412.422062:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767412.422072:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967319936/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767412.422078:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767412.422081:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767412.422082:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767412.422085:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4034 00000100:00080000:1.0:1775767412.422098:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767412.422102:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:0.0:1775767412.422106:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:1.0:1775767412.422108:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767412.422123:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967320064/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767412.422145:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767412/real 1775767412] req@ffff8f994f2e4700 x1862022967319680/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767412.422160:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967319680/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767412.422168:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767412.422172:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767412.422175:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967319680:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767412.422192:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967319808:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775767412.422206:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767412.422214:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767412/real 1775767412] req@ffff8f994f2e6300 x1862022967319808/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767412.422222:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967319808/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767412.422228:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767412.422230:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767412.422233:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967319808:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767412.422236:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967319936:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767412.422247:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767412.422253:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767412/real 1775767412] req@ffff8f994f2e4e00 x1862022967319936/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767412.422260:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967319936/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767412.422275:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767412.422277:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767412.422282:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967319936:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767412.422286:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967320064:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767412.422297:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767412.422303:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767412/real 1775767412] req@ffff8f994f2e7b80 x1862022967320064/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767412.422311:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967320064/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767412.422315:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767412.422317:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767412.422320:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967320064:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767413.317872:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767413.317936:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767414.341820:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767414.341909:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767415.365828:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767415.365891:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767416.389859:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767416.389918:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767417.413851:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767417.413896:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767417.541883:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767417.541895:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767417.541901:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767417.541977:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4039 00000100:00080000:2.0:1775767417.541988:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767417.541996:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767417.542033:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0e00 x1862022967320192/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767417.542072:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767417.542076:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767417.542078:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767417.542082:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4039 00000100:00080000:2.0:1775767417.542086:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767417.542090:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767417.542100:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767417.542116:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2d80 x1862022967320320/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767417.542127:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967320192:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775767417.542130:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767417.542136:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767417.542138:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767417.542143:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4039 00000100:00080000:2.0:1775767417.542147:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767417.542151:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767417.542157:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767417.542172:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1500 x1862022967320448/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767417.542195:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775767417.542208:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767417.542212:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767417.542214:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767417.542219:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4039 00000100:00080000:2.0:1775767417.542223:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767417.542226:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767417.542231:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767417.542240:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767417/real 1775767417] req@ffff8f99703d0e00 x1862022967320192/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767417.542243:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0700 x1862022967320576/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767417.542270:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0e00 x1862022967320192/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767417.542280:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767417.542283:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767417.542286:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967320192:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767417.542306:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967320320:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775767417.542320:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767417.542328:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767417/real 1775767417] req@ffff8f99703d2d80 x1862022967320320/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767417.542336:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2d80 x1862022967320320/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767417.542341:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767417.542343:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767417.542346:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967320320:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767417.542350:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967320448:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767417.542360:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767417.542365:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767417/real 1775767417] req@ffff8f99703d1500 x1862022967320448/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767417.542372:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1500 x1862022967320448/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767417.542377:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767417.542379:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767417.542381:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967320448:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767417.542384:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967320576:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767417.542393:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767417.542398:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767417/real 1775767417] req@ffff8f99703d0700 x1862022967320576/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767417.542405:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0700 x1862022967320576/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767417.542410:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767417.542412:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767417.542413:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967320576:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767418.437865:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767418.437933:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767419.461995:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767419.462044:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:0.0:1775767420.485897:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767420.485928:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767421.509891:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767421.510054:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767422.533838:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767422.533874:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767422.661889:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767422.661897:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767422.661901:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767422.661910:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4044 00000100:00080000:3.0:1775767422.661920:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767422.661925:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767422.661957:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd180 x1862022967320704/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767422.661980:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767422.661983:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767422.661984:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767422.661987:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4044 00000100:00080000:3.0:1775767422.661989:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767422.661992:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767422.661998:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767422.662006:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddf80 x1862022967320832/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767422.662013:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767422.662016:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767422.662017:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767422.662020:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4044 00000100:00100000:0.0:1775767422.662034:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967320704:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775767422.662054:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767422.662058:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767422.662063:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767422.662072:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967320960/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767422.662080:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767422.662082:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767422.662083:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1775767422.662089:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775767422.662089:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4044 00000100:00080000:3.0:1775767422.662095:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767422.662098:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767422.662101:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767422.662111:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967321088/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767422.662140:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767422/real 1775767422] req@ffff8f9970edd180 x1862022967320704/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767422.662160:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd180 x1862022967320704/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767422.662172:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767422.662177:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767422.662182:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967320704:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767422.662189:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967320832:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775767422.662210:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767422.662222:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767422/real 1775767422] req@ffff8f9970eddf80 x1862022967320832/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767422.662234:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddf80 x1862022967320832/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767422.662243:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767422.662247:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767422.662250:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967320832:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767422.662276:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967320960:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767422.662427:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767422.662452:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767422/real 1775767422] req@ffff8f9970edea00 x1862022967320960/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767422.662465:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967320960/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767422.662474:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767422.662477:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767422.662480:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967320960:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767422.662488:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967321088:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767422.662500:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767422.662507:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767422/real 1775767422] req@ffff8f9970edca80 x1862022967321088/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767422.662514:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967321088/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767422.662520:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767422.662522:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767422.662524:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967321088:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767423.557851:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767423.557911:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767424.581852:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767424.581906:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767425.605859:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767425.605922:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767426.629845:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767426.629908:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767427.653862:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767427.653937:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767427.781866:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767427.781874:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767427.781878:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767427.781887:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4049 00000100:00080000:3.0:1775767427.781896:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767427.781902:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767427.781932:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967321216/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767427.781963:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767427.782003:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767427.782006:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767427.782012:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4049 00000100:00080000:3.0:1775767427.782016:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775767427.782019:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967321216:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775767427.782022:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767427.782030:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767427.782041:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967321344/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767427.782053:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767427.782060:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767427.782063:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767427.782067:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4049 00000100:00080000:3.0:1775767427.782071:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767427.782075:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767427.782081:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767427.782095:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967321472/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767427.782105:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767427.782109:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:0.0:1775767427.782110:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775767427.782111:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767427.782116:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4049 00000100:00080000:3.0:1775767427.782120:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767427.782124:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767427.782141:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767427.782150:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967321600/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767427.782153:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767427/real 1775767427] req@ffff8f9970edc380 x1862022967321216/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767427.782183:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967321216/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767427.782192:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767427.782196:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767427.782198:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967321216:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767427.782216:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967321344:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775767427.782230:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767427.782238:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767427/real 1775767427] req@ffff8f9970edfb80 x1862022967321344/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767427.782246:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967321344/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767427.782251:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767427.782253:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767427.782256:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967321344:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767427.782262:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967321472:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767427.782280:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767427.782291:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767427/real 1775767427] req@ffff8f9970ede300 x1862022967321472/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767427.782311:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967321472/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767427.782320:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767427.782323:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767427.782326:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967321472:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767427.782333:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967321600:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767427.782354:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767427.782372:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767427/real 1775767427] req@ffff8f9970edd880 x1862022967321600/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767427.782380:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967321600/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767427.782388:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767427.782391:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767427.782393:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967321600:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767428.677839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767428.677897:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767429.701833:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767429.701882:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767430.725813:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767430.725882:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767431.749866:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775767431.749891:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767432.773884:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767432.773895:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767432.901885:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767432.901895:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767432.901900:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767432.901910:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4054 00000100:00080000:1.0:1775767432.901923:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767432.901932:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767432.901982:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967321728/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767432.902012:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767432.902016:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767432.902017:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767432.902021:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4054 00000100:00080000:1.0:1775767432.902024:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767432.902028:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767432.902037:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767432.902052:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967321856/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767432.902060:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767432.902062:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1775767432.902083:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967321728:192.168.202.146@tcp:250:kworker.0 00000800:00000100:0.0:1775767432.902142:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775767432.902167:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767432.902175:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4054 00000100:00100000:0.0:1775767432.902176:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767432/real 1775767432] req@ffff8f994f2e7b80 x1862022967321728/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767432.902183:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767432.902191:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767432.902192:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967321728/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767432.902201:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1775767432.902202:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1775767432.902204:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767432.902208:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967321728:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767432.902213:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967321856:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767432.902226:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967321984/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767432.902230:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767432.902240:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767432/real 1775767432] req@ffff8f994f2e4e00 x1862022967321856/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767432.902242:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767432.902249:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767432.902253:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775767432.902256:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967321856/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767432.902259:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4054 00000100:00080000:1.0:1775767432.902267:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767432.902268:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767432.902273:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00080000:1.0:1775767432.902275:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767432.902278:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967321856:192.168.202.146@tcp:38:kworker.0 00000008:00100000:1.0:1775767432.902284:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767432.902298:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967322112/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767432.902302:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967321984:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767432.902326:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767432.902337:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767432/real 1775767432] req@ffff8f994f2e6300 x1862022967321984/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767432.902348:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967321984/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767432.902361:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767432.902363:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767432.902366:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967321984:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767432.902375:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967322112:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767432.902388:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767432.902394:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767432/real 1775767432] req@ffff8f994f2e4700 x1862022967322112/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767432.902400:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967322112/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767432.902404:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767432.902406:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767432.902408:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967322112:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767433.797831:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767433.797861:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767434.821832:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767434.821874:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767435.845820:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767435.845881:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767436.869883:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775767436.869951:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767437.893805:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767437.893880:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767438.021848:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767438.021859:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767438.021864:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767438.021874:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4059 00000100:00080000:3.0:1775767438.021886:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767438.021894:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767438.021930:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967322240/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767438.021952:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767438.021955:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767438.021956:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767438.021960:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4059 00000100:00080000:3.0:1775767438.021963:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767438.021965:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767438.021973:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767438.021981:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967322368/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767438.021989:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767438.021992:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767438.021993:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775767438.021997:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967322240:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775767438.021999:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4059 00000100:00080000:3.0:1775767438.022037:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767438.022040:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767438.022045:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775767438.022052:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767438.022057:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967322496/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767438.022064:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767438.022067:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767438.022068:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767438.022071:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4059 00000100:00080000:3.0:1775767438.022073:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767438.022075:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767438.022078:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767438.022085:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967322624/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767438.022095:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767438/real 1775767438] req@ffff8f9970edf480 x1862022967322240/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767438.022116:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967322240/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767438.022127:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767438.022131:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767438.022136:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967322240:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767438.022158:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967322368:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775767438.022175:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767438.022184:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767438/real 1775767438] req@ffff8f9970edf800 x1862022967322368/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767438.022192:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967322368/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767438.022197:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767438.022200:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767438.022202:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967322368:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767438.022207:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967322496:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767438.022225:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767438.022235:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767438/real 1775767438] req@ffff8f9970edc000 x1862022967322496/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767438.022247:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967322496/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767438.022256:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767438.022259:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767438.022263:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967322496:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767438.022292:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967322624:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767438.022309:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767438.022317:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767438/real 1775767438] req@ffff8f9970ede680 x1862022967322624/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767438.022328:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967322624/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767438.022336:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767438.022339:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767438.022342:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967322624:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767438.917848:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767438.917919:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767439.941870:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767439.941914:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767440.965857:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767440.965934:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767441.989836:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:2.0:1775767441.989880:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767443.013845:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767443.013898:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767443.141826:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767443.141837:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767443.141842:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767443.141853:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4064 00000100:00080000:3.0:1775767443.141864:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767443.141872:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767443.141904:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967322752/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767443.141929:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775767443.141965:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967322752:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775767443.141973:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767443.141975:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767443.141980:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4064 00000100:00080000:3.0:1775767443.141984:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767443.141990:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767443.141998:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767443.142011:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967322880/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767443.142020:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767443.142023:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767443.142025:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767443.142028:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4064 00000100:00080000:3.0:1775767443.142031:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767443.142034:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767443.142037:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767443.142045:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf100 x1862022967323008/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767443.142051:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767443.142053:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767443.142055:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767443.142057:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4064 00000100:00080000:3.0:1775767443.142059:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767443.142062:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767443.142075:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767443.142077:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967322880:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767443.142098:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967323008:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767443.142144:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967323136/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767443.142183:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967323136:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767444.037819:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767444.037865:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767445.061836:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767445.061904:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767446.085884:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767446.085960:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1775767446.213968:0:2366:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:0.0:1775767446.214005:0:2366:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1775767446.214019:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775767446.214057:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775767446.214068:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775767446.214076:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:1.0:1775767446.214107:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767443/real 1775767446] req@ffff8f9970edc700 x1862022967323136/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767446.214128:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967323136/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767446.214137:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767446.214141:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767446.214144:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967323136:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767446.214149:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767443/real 1775767446] req@ffff8f9970eded80 x1862022967322880/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767446.214157:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967322880/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767446.214162:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767446.214164:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767446.214166:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967322880:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767446.214170:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767443/real 1775767446] req@ffff8f9970edf100 x1862022967323008/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767446.214178:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf100 x1862022967323008/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767446.214183:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767446.214185:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767446.214186:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967323008:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767446.214191:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767443/real 1775767446] req@ffff8f9970edd500 x1862022967322752/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767446.214198:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967322752/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767446.214202:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767446.214204:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767446.214206:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967322752:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767447.109899:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767447.109953:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767448.133862:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767448.133899:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767448.261854:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767448.261863:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767448.261875:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767448.261885:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4069 00000100:00080000:2.0:1775767448.261895:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767448.261901:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767448.261935:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3800 x1862022967323264/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767448.261959:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767448.261961:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767448.261963:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767448.262015:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4069 00000100:00080000:2.0:1775767448.262020:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767448.262025:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767448.262036:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767448.262050:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0380 x1862022967323392/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767448.262057:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967323264:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775767448.262064:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767448.262068:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767448.262070:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767448.262074:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4069 00000100:00080000:2.0:1775767448.262078:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767448.262082:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767448.262087:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767448.262099:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0000 x1862022967323520/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767448.262109:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767448.262112:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767448.262113:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767448.262116:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4069 00000100:00080000:2.0:1775767448.262121:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767448.262125:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767448.262131:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767448.262146:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2680 x1862022967323648/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767448.262160:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767448.262222:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767448/real 1775767448] req@ffff8f99703d3800 x1862022967323264/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767448.262244:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3800 x1862022967323264/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767448.262258:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767448.262265:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767448.262272:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967323264:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767448.262305:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967323392:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775767448.262330:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767448.262345:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767448/real 1775767448] req@ffff8f99703d0380 x1862022967323392/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767448.262357:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0380 x1862022967323392/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767448.262366:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767448.262370:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767448.262373:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967323392:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767448.262379:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967323520:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767448.262398:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767448.262408:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767448/real 1775767448] req@ffff8f99703d0000 x1862022967323520/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767448.262420:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0000 x1862022967323520/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767448.262428:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767448.262431:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767448.262434:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967323520:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767448.262441:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967323648:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767448.262482:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767448.262494:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767448/real 1775767448] req@ffff8f99703d2680 x1862022967323648/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767448.262505:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2680 x1862022967323648/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767448.262513:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767448.262517:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767448.262520:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967323648:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767449.157844:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767449.157988:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767450.181838:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767450.181920:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767451.205825:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767451.205885:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767452.229816:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767452.229890:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767453.253865:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767453.253899:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767453.381807:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767453.381816:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767453.381820:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767453.381829:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4074 00000100:00080000:2.0:1775767453.381838:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767453.381843:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767453.381874:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2680 x1862022967323776/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767453.381897:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767453.381900:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767453.381902:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767453.381905:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4074 00000100:00080000:2.0:1775767453.381907:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767453.381909:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767453.381917:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767453.381924:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0000 x1862022967323904/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767453.381932:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767453.381935:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767453.381937:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767453.381939:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4074 00000100:00080000:2.0:1775767453.381941:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767453.381944:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767453.381954:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967323776:192.168.202.146@tcp:250:kworker.0 00000008:00100000:2.0:1775767453.381982:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767453.381992:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0380 x1862022967324032/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767453.382000:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767453.382002:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767453.382003:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767453.382006:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4074 00000100:00080000:2.0:1775767453.382009:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767453.382012:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767453.382015:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775767453.382020:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767453.382024:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3800 x1862022967324160/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767453.382058:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767453/real 1775767453] req@ffff8f99703d2680 x1862022967323776/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767453.382072:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2680 x1862022967323776/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767453.382091:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767453.382094:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767453.382097:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967323776:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767453.382102:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967323904:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775767453.382118:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767453.382125:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767453/real 1775767453] req@ffff8f99703d0000 x1862022967323904/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767453.382133:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0000 x1862022967323904/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767453.382139:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767453.382141:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767453.382143:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967323904:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767453.382160:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967324032:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767453.382171:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767453.382177:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767453/real 1775767453] req@ffff8f99703d0380 x1862022967324032/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767453.382184:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0380 x1862022967324032/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767453.382189:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767453.382191:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767453.382193:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967324032:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767453.382197:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967324160:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767453.382206:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767453.382212:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767453/real 1775767453] req@ffff8f99703d3800 x1862022967324160/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767453.382224:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3800 x1862022967324160/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767453.382229:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767453.382231:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767453.382233:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967324160:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767454.277810:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767454.277880:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767455.301851:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767455.301931:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767456.325812:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767456.325872:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767457.349841:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767457.349896:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767458.373848:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767458.373934:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767458.501828:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767458.501838:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767458.501843:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767458.501853:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4080 00000100:00080000:2.0:1775767458.501864:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767458.501871:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767458.501909:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0700 x1862022967324288/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767458.501939:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767458.501943:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767458.501945:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767458.501950:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4080 00000100:00100000:1.0:1775767458.501980:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967324288:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775767458.501993:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767458.501999:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767458.502009:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767458.502023:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1500 x1862022967324416/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767458.502035:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767458.502040:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767458.502044:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1775767458.502045:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775767458.502052:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4080 00000100:00080000:2.0:1775767458.502055:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767458.502059:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767458.502063:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767458.502072:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2d80 x1862022967324544/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767458.502079:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767458.502082:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767458.502083:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767458.502085:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4080 00000100:00080000:2.0:1775767458.502088:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767458.502090:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767458.502093:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767458.502100:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0e00 x1862022967324672/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767458.502105:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767458/real 1775767458] req@ffff8f99703d0700 x1862022967324288/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767458.502129:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0700 x1862022967324288/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767458.502139:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767458.502143:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767458.502147:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967324288:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767458.502165:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967324416:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775767458.502184:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767458.502193:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767458/real 1775767458] req@ffff8f99703d1500 x1862022967324416/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767458.502220:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1500 x1862022967324416/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767458.502227:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767458.502231:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767458.502234:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967324416:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767458.502241:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967324544:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767458.502263:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767458.502275:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767458/real 1775767458] req@ffff8f99703d2d80 x1862022967324544/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767458.502288:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2d80 x1862022967324544/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767458.502294:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767458.502296:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767458.502298:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967324544:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767458.502302:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967324672:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767458.502317:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767458.502323:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767458/real 1775767458] req@ffff8f99703d0e00 x1862022967324672/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767458.502330:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0e00 x1862022967324672/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767458.502340:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767458.502342:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767458.502343:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967324672:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767459.397818:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767459.397885:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767460.421849:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767460.421915:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767461.445806:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767461.445871:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767462.469841:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767462.469911:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767463.493834:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767463.493898:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767463.621857:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767463.621869:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767463.621874:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767463.621884:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4085 00000100:00080000:0.0:1775767463.621896:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767463.621903:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767463.621936:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967324800/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767463.621961:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767463.621963:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767463.621965:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767463.621968:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4085 00000100:00080000:0.0:1775767463.621971:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767463.621974:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767463.621982:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767463.621990:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967324928/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767463.621998:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767463.622001:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767463.622002:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767463.622004:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4085 00000100:00080000:0.0:1775767463.622006:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767463.622009:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767463.622013:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767463.622018:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967324800:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767463.622023:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967325056/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767463.622031:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767463.622033:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767463.622034:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767463.622036:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4085 00000100:00080000:0.0:1775767463.622038:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767463.622040:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767463.622043:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767463.622051:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967325184/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767463.622077:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767463.622111:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767463/real 1775767463] req@ffff8f9970a74700 x1862022967324800/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767463.622124:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967324800/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767463.622131:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767463.622134:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767463.622137:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967324800:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767463.622142:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967324928:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775767463.622156:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767463.622163:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767463/real 1775767463] req@ffff8f9970a76d80 x1862022967324928/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767463.622171:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967324928/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767463.622176:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767463.622178:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767463.622181:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967324928:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767463.622199:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967325056:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767463.622210:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767463.622215:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767463/real 1775767463] req@ffff8f9970a77800 x1862022967325056/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767463.622222:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967325056/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767463.622227:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767463.622229:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767463.622231:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967325056:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767463.622235:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967325184:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767463.622244:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767463.622250:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767463/real 1775767463] req@ffff8f9970a74380 x1862022967325184/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767463.622259:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967325184/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767463.622268:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767463.622270:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767463.622317:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967325184:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767464.517871:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767464.517942:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767465.541835:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767465.541901:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767466.565858:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767466.565937:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767467.589858:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767467.589911:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767468.613902:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767468.613902:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00080000:1.0:1775767468.742037:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767468.742053:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767468.742061:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767468.742076:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4090 00000100:00080000:1.0:1775767468.742095:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767468.742104:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767468.742165:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967325312/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767468.742206:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767468.742210:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767468.742213:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767468.742218:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4090 00000100:00080000:1.0:1775767468.742223:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767468.742228:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767468.742244:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767468.742256:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967325312:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767468.742274:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967325440/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767468.742288:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767468.742295:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767468.742297:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767468.742301:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4090 00000100:00080000:1.0:1775767468.742306:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767468.742312:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767468.742318:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767468.742337:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967325568/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767468.742360:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767468.742365:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767468.742367:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767468.742372:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4090 00000100:00080000:1.0:1775767468.742377:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767468.742381:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767468.742386:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775767468.742387:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767468.742406:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967325696/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767468.742446:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767468/real 1775767468] req@ffff8f994f2e6a00 x1862022967325312/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767468.742470:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967325312/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767468.742482:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767468.742487:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767468.742491:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967325312:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767468.742525:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967325440:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775767468.742558:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767468.742572:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767468/real 1775767468] req@ffff8f994f2e5f80 x1862022967325440/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767468.742581:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967325440/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767468.742588:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767468.742591:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767468.742593:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967325440:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767468.742597:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967325568:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767468.742659:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767468.742667:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767468/real 1775767468] req@ffff8f994f2e4000 x1862022967325568/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767468.742690:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967325568/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767468.742699:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767468.742702:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767468.742706:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967325568:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767468.742713:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967325696:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767468.742735:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767468.742773:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767468/real 1775767468] req@ffff8f994f2e6680 x1862022967325696/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767468.742784:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967325696/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767468.742790:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767468.742792:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767468.742795:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967325696:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767469.637859:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767469.637963:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767470.661891:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767470.661961:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767471.685845:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767471.685937:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767472.709864:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767472.709925:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767473.733819:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767473.733893:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767473.861855:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767473.861866:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767473.861872:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767473.861884:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4095 00000100:00080000:1.0:1775767473.861897:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767473.861904:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767473.861949:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967325824/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767473.861987:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767473.861991:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767473.861993:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767473.861998:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4095 00000100:00080000:1.0:1775767473.862002:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767473.862007:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767473.862018:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767473.862031:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967325952/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767473.862039:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767473.862042:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767473.862044:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767473.862049:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4095 00000100:00100000:0.0:1775767473.862052:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967325824:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775767473.862053:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767473.862057:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767473.862065:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767473.862071:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967326080/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767473.862140:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767473.862187:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767473/real 1775767473] req@ffff8f994f2e6d80 x1862022967325824/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767473.862217:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767473.862229:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767473.862232:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767473.862244:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4095 00000100:00100000:0.0:1775767473.862250:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967325824/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767473.862254:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767473.862268:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767473.862274:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00080000:1.0:1775767473.862278:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767473.862284:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967325824:192.168.202.146@tcp:250:kworker.0 00000008:00100000:1.0:1775767473.862293:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767473.862338:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967326208/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767473.862344:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967325952:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775767473.862377:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767473.862392:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767473/real 1775767473] req@ffff8f994f2e4a80 x1862022967325952/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767473.862402:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967325952/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767473.862409:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767473.862411:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767473.862414:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967325952:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767473.862421:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967326080:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767473.862441:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767473.862453:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767473/real 1775767473] req@ffff8f994f2e7800 x1862022967326080/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767473.862466:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967326080/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767473.862482:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767473.862484:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767473.862487:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967326080:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767473.862500:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967326208:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767473.862516:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767473.862524:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767473/real 1775767473] req@ffff8f994f2e5180 x1862022967326208/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767473.862531:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967326208/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767473.862537:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767473.862540:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767473.862542:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967326208:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767474.757885:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767474.757969:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767475.781882:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767475.781928:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767476.805857:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767476.805926:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767477.829901:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767477.829916:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767478.853804:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767478.853905:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767478.981870:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767478.981880:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767478.981885:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767478.981897:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4100 00000100:00080000:2.0:1775767478.981909:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767478.981917:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767478.981953:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3100 x1862022967326336/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767478.981983:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767478.981987:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767478.981990:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767478.981995:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4100 00000100:00080000:2.0:1775767478.982000:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767478.982005:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767478.982015:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767478.982031:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2300 x1862022967326464/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767478.982037:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967326336:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775767478.982082:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767478.982086:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767478.982088:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1775767478.982092:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775767478.982095:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4100 00000100:00080000:2.0:1775767478.982101:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767478.982105:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767478.982112:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767478.982125:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1180 x1862022967326592/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767478.982135:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767478.982159:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767478.982161:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767478.982165:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4100 00000100:00080000:2.0:1775767478.982170:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767478.982174:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767478.982180:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767478.982189:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3480 x1862022967326720/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767478.982235:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767478/real 1775767478] req@ffff8f99703d3100 x1862022967326336/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767478.982269:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3100 x1862022967326336/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767478.982279:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767478.982283:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767478.982287:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967326336:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767478.982309:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967326464:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767478.982432:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767478.982445:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767478/real 1775767478] req@ffff8f99703d2300 x1862022967326464/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767478.982455:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2300 x1862022967326464/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767478.982462:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767478.982465:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767478.982519:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967326464:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767478.982526:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967326592:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767478.982552:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767478.982565:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767478/real 1775767478] req@ffff8f99703d1180 x1862022967326592/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767478.982577:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1180 x1862022967326592/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767478.982585:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767478.982588:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767478.982592:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967326592:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767478.982598:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967326720:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767478.982620:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767478.982629:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767478/real 1775767478] req@ffff8f99703d3480 x1862022967326720/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767478.982637:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3480 x1862022967326720/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767478.982643:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767478.982645:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767478.982647:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967326720:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767479.877850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767479.877920:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767480.901819:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767480.901900:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767481.925839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767481.925907:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767482.949806:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767482.949895:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767483.973844:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767483.973917:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767484.101836:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767484.101848:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767484.101852:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767484.101862:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4105 00000100:00080000:2.0:1775767484.101874:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767484.101880:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767484.101921:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3b80 x1862022967326848/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767484.101944:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767484.101947:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767484.101948:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767484.101951:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4105 00000100:00080000:2.0:1775767484.101955:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767484.101958:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767484.101967:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767484.101979:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1880 x1862022967326976/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767484.101979:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967326848:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775767484.101993:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767484.101995:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767484.101996:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767484.101999:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4105 00000100:00080000:2.0:1775767484.102001:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767484.102003:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767484.102007:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775767484.102038:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767484.102052:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2a00 x1862022967327104/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767484.102060:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767484.102062:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767484.102063:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767484.102066:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4105 00000100:00080000:2.0:1775767484.102069:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767484.102071:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767484.102074:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767484.102083:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1c00 x1862022967327232/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767484.102089:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767484/real 1775767484] req@ffff8f99703d3b80 x1862022967326848/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767484.102120:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3b80 x1862022967326848/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767484.102132:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767484.102136:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767484.102140:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967326848:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767484.102164:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967326976:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767484.102214:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767484.102229:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767484/real 1775767484] req@ffff8f99703d1880 x1862022967326976/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767484.102242:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1880 x1862022967326976/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767484.102250:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767484.102253:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767484.102256:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967326976:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767484.102262:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967327104:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767484.102282:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767484.102293:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767484/real 1775767484] req@ffff8f99703d2a00 x1862022967327104/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767484.102304:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2a00 x1862022967327104/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767484.102319:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767484.102322:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767484.102326:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967327104:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767484.102332:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967327232:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767484.102350:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767484.102361:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767484/real 1775767484] req@ffff8f99703d1c00 x1862022967327232/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767484.102374:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1c00 x1862022967327232/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767484.102382:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767484.102385:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767484.102388:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967327232:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767484.997904:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767484.997939:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767486.021863:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767486.021955:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767487.045912:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767487.045979:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767488.069865:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767488.069962:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767489.093818:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767489.093873:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767489.221852:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767489.221861:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767489.221866:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767489.221877:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4110 00000100:00080000:0.0:1775767489.221889:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767489.221894:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767489.221924:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967327360/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767489.221947:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767489.221950:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767489.221952:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767489.221955:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4110 00000100:00080000:0.0:1775767489.221957:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767489.221960:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767489.221968:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767489.221976:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967327488/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767489.221984:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767489.221987:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767489.221989:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775767489.221991:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967327360:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775767489.221993:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4110 00000100:00080000:0.0:1775767489.221997:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767489.222003:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767489.222006:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767489.222013:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967327616/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767489.222019:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767489.222021:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767489.222022:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767489.222025:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4110 00000100:00080000:0.0:1775767489.222027:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767489.222029:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767489.222033:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767489.222039:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967327744/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775767489.222067:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767489.222119:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767489/real 1775767489] req@ffff8f9970a76a00 x1862022967327360/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767489.222141:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967327360/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767489.222168:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767489.222174:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767489.222178:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967327360:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767489.222201:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967327488:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767489.222227:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767489.222240:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767489/real 1775767489] req@ffff8f9970a77100 x1862022967327488/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767489.222253:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967327488/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767489.222261:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767489.222273:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767489.222277:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967327488:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767489.222283:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967327616:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767489.222303:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767489.222313:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767489/real 1775767489] req@ffff8f9970a75c00 x1862022967327616/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767489.222324:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967327616/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767489.222332:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767489.222336:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767489.222339:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967327616:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767489.222345:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967327744:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767489.222363:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767489.222373:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767489/real 1775767489] req@ffff8f9970a77480 x1862022967327744/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767489.222425:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967327744/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767489.222434:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767489.222437:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767489.222441:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967327744:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767490.117831:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767490.117896:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767491.141945:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767491.141998:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775767492.165900:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767492.165916:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767493.189823:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767493.189870:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767494.213840:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767494.213913:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767494.341836:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767494.341845:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767494.341850:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767494.341860:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4115 00000100:00080000:1.0:1775767494.341872:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767494.341879:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767494.341920:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967327872/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767494.341949:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767494.341953:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767494.341955:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767494.341959:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4115 00000100:00080000:1.0:1775767494.341971:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767494.341976:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767494.341982:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967327872:192.168.202.146@tcp:250:kworker.0 00000008:00100000:1.0:1775767494.341989:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767494.342012:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967328000/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767494.342025:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767494.342028:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767494.342030:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767494.342035:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4115 00000100:00080000:1.0:1775767494.342039:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767494.342043:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:3.0:1775767494.342048:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:1.0:1775767494.342051:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767494.342066:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967328128/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767494.342076:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767494.342080:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767494.342082:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767494.342086:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4115 00000100:00080000:1.0:1775767494.342091:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775767494.342096:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767494/real 1775767494] req@ffff8f994f2e5180 x1862022967327872/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767494.342097:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767494.342116:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767494.342131:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967328256/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767494.342156:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967327872/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767494.342166:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767494.342169:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767494.342173:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967327872:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767494.342194:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967328000:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767494.342212:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767494.342223:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767494/real 1775767494] req@ffff8f994f2e7800 x1862022967328000/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767494.342236:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967328000/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767494.342292:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767494.342295:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767494.342297:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967328000:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767494.342302:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967328128:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767494.342319:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767494.342327:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767494/real 1775767494] req@ffff8f994f2e4a80 x1862022967328128/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767494.342335:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967328128/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767494.342340:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767494.342342:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767494.342344:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967328128:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767494.342348:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967328256:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767494.342357:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767494.342363:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767494/real 1775767494] req@ffff8f994f2e6d80 x1862022967328256/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767494.342370:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967328256/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767494.342375:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767494.342376:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767494.342391:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967328256:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767495.237829:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767495.237847:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767496.261863:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767496.261884:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767497.285844:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767497.285925:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767498.309827:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767498.309877:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767499.333912:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767499.333912:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00080000:2.0:1775767499.461864:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767499.461897:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767499.461901:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767499.461912:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4121 00000100:00080000:2.0:1775767499.461922:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767499.461927:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767499.461961:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1c00 x1862022967328384/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767499.461989:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767499.461992:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767499.461993:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767499.461997:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4121 00000100:00080000:2.0:1775767499.462000:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767499.462003:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767499.462012:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767499.462023:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2a00 x1862022967328512/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767499.462030:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967328384:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775767499.462035:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767499.462047:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767499.462065:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767499.462071:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4121 00000100:00080000:2.0:1775767499.462076:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767499.462080:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767499.462086:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767499.462099:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1880 x1862022967328640/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767499.462122:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767499.462126:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767499.462128:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767499.462131:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4121 00000100:00080000:2.0:1775767499.462134:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767499.462142:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:3.0:1775767499.462143:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:2.0:1775767499.462152:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767499.462167:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3b80 x1862022967328768/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767499.462201:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767499/real 1775767499] req@ffff8f99703d1c00 x1862022967328384/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767499.462227:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1c00 x1862022967328384/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767499.462236:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767499.462239:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767499.462243:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967328384:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767499.462262:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967328512:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767499.462283:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767499.462309:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767499/real 1775767499] req@ffff8f99703d2a00 x1862022967328512/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767499.462318:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2a00 x1862022967328512/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767499.462324:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767499.462327:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767499.462329:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967328512:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767499.462333:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967328640:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767499.462346:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767499.462379:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767499/real 1775767499] req@ffff8f99703d1880 x1862022967328640/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767499.462388:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1880 x1862022967328640/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767499.462394:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767499.462396:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767499.462398:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967328640:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767499.462402:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967328768:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767499.462416:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767499.462422:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767499/real 1775767499] req@ffff8f99703d3b80 x1862022967328768/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767499.462430:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3b80 x1862022967328768/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767499.462436:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767499.462437:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767499.462439:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967328768:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767500.357820:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767500.357873:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767501.381853:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767501.381914:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767502.405914:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767502.406001:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775767503.429816:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767503.429865:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767504.453824:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767504.453904:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767504.581844:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767504.581852:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767504.581856:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767504.581865:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4126 00000100:00080000:0.0:1775767504.581874:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767504.581880:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767504.581911:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967328896/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767504.581934:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767504.581936:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767504.581938:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767504.581941:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4126 00000100:00080000:0.0:1775767504.581943:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767504.581946:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767504.581960:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767504.581968:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967328896:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767504.581971:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967329024/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767504.581983:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767504.581985:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767504.581987:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767504.581989:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4126 00000100:00080000:0.0:1775767504.581993:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767504.581995:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767504.582018:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767504.582030:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967329152/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767504.582036:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767504.582039:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767504.582041:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767504.582043:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4126 00000100:00080000:0.0:1775767504.582046:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767504.582049:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767504.582052:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775767504.582057:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767504.582065:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967329280/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767504.582106:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767504/real 1775767504] req@ffff8f9970a75500 x1862022967328896/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767504.582122:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967328896/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767504.582170:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767504.582175:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767504.582180:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967328896:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767504.582204:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967329024:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767504.582227:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767504.582237:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767504/real 1775767504] req@ffff8f9970a76300 x1862022967329024/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767504.582245:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967329024/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767504.582251:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767504.582253:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767504.582255:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967329024:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767504.582259:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967329152:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767504.582270:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767504.582293:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767504/real 1775767504] req@ffff8f9970a75180 x1862022967329152/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767504.582301:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967329152/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767504.582307:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767504.582308:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767504.582310:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967329152:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767504.582330:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967329280:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767504.582344:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767504.582352:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767504/real 1775767504] req@ffff8f9970a77480 x1862022967329280/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767504.582359:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967329280/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767504.582365:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767504.582367:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767504.582370:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967329280:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767505.477846:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767505.477954:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767506.501897:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767506.501928:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767507.525853:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767507.525902:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767508.549839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767508.549915:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767509.573830:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767509.573888:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767509.701886:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767509.701897:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767509.701901:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767509.701912:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4131 00000100:00080000:1.0:1775767509.701925:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767509.701932:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767509.701999:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967329408/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767509.702026:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767509.702030:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767509.702048:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767509.702052:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4131 00000100:00080000:1.0:1775767509.702057:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767509.702062:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767509.702073:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767509.702080:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967329408:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767509.702088:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967329536/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767509.702101:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767509.702106:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767509.702108:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767509.702112:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4131 00000100:00080000:1.0:1775767509.702117:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767509.702121:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767509.702127:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767509.702180:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967329536:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767509.702188:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967329664/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767509.702202:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767509.702207:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767509.702209:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767509.702214:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4131 00000100:00100000:3.0:1775767509.702218:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967329664:192.168.202.146@tcp:8:kworker.0 00000100:00080000:1.0:1775767509.702220:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767509.702229:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767509.702236:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767509.702261:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967329792/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767509.702322:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967329792:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767510.597811:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767510.597855:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767511.621839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767511.621903:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767512.645824:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767512.645962:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775767512.774019:0:2368:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:2.0:1775767512.774064:0:2368:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775767512.774079:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:2.0:1775767512.774117:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:2.0:1775767512.774124:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:2.0:1775767512.774131:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:3.0:1775767512.774185:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767509/real 1775767512] req@ffff8f994f2e6a00 x1862022967329792/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767512.774226:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967329792/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767512.774240:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767512.774246:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767512.774250:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967329792:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767512.774981:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767509/real 1775767512] req@ffff8f994f2e5f80 x1862022967329664/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767512.775021:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967329664/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767512.775035:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767512.775041:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767512.775045:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967329664:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767512.775053:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767509/real 1775767512] req@ffff8f994f2e4000 x1862022967329536/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767512.775065:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967329536/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767512.775090:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767512.775094:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767512.775097:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967329536:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767512.775109:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767509/real 1775767512] req@ffff8f994f2e6680 x1862022967329408/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767512.775122:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967329408/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767512.775130:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767512.775133:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767512.775136:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967329408:192.168.202.146@tcp:250:kworker.0 00010000:00010000:1.0:1775767513.669929:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767513.669929:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775767514.693835:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767514.693880:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767514.821873:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767514.821880:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767514.821884:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767514.821893:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4136 00000100:00080000:2.0:1775767514.821917:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767514.821923:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767514.821959:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3480 x1862022967329920/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767514.821989:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767514.821993:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767514.821995:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767514.822000:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4136 00000100:00080000:2.0:1775767514.822003:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767514.822008:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767514.822019:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767514.822037:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1180 x1862022967330048/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767514.822047:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767514.822050:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767514.822052:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767514.822056:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4136 00000100:00080000:2.0:1775767514.822061:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775767514.822066:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967329920:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775767514.822068:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767514.822078:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767514.822093:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2300 x1862022967330176/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767514.822105:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767514.822108:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767514.822110:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767514.822114:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4136 00000100:00080000:2.0:1775767514.822117:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767514.822121:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767514.822126:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767514.822139:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3100 x1862022967330304/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775767514.822164:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767514.822218:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767514/real 1775767514] req@ffff8f99703d3480 x1862022967329920/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767514.822241:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3480 x1862022967329920/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767514.822253:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767514.822259:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767514.822278:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967329920:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767514.822286:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967330048:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767514.822313:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767514.822330:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767514/real 1775767514] req@ffff8f99703d1180 x1862022967330048/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767514.822343:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1180 x1862022967330048/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767514.822353:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767514.822358:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767514.822361:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967330048:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767514.822390:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967330176:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767514.822412:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767514.822423:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767514/real 1775767514] req@ffff8f99703d2300 x1862022967330176/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767514.822455:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2300 x1862022967330176/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767514.822464:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767514.822467:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767514.822471:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967330176:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767514.822477:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967330304:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767514.822499:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767514.822510:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767514/real 1775767514] req@ffff8f99703d3100 x1862022967330304/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767514.822522:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3100 x1862022967330304/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767514.822530:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767514.822533:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767514.822536:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967330304:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767515.717833:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767515.717952:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767516.741906:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767516.742007:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767517.765841:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767517.765937:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767518.789914:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767518.790000:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767519.813821:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767519.813888:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767519.941830:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767519.941838:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767519.941843:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767519.941866:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4141 00000100:00080000:2.0:1775767519.941881:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767519.941904:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767519.941946:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0e00 x1862022967330432/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767519.941971:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767519.941974:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767519.941976:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767519.941980:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4141 00000100:00080000:2.0:1775767519.941983:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767519.941986:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767519.941994:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767519.942004:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2d80 x1862022967330560/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767519.942015:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775767519.942015:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967330432:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775767519.942025:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767519.942029:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767519.942035:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4141 00000100:00080000:2.0:1775767519.942040:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767519.942045:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767519.942052:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767519.942068:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1500 x1862022967330688/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767519.942081:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767519.942085:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767519.942087:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1775767519.942092:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775767519.942097:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4141 00000100:00080000:2.0:1775767519.942101:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767519.942105:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767519.942112:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767519.942122:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0700 x1862022967330816/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767519.942162:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767519/real 1775767519] req@ffff8f99703d0e00 x1862022967330432/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767519.942196:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0e00 x1862022967330432/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767519.942276:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767519.942281:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767519.942285:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967330432:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767519.942306:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967330560:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767519.942346:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767519.942357:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767519/real 1775767519] req@ffff8f99703d2d80 x1862022967330560/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767519.942367:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2d80 x1862022967330560/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767519.942373:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767519.942376:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767519.942378:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967330560:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767519.942383:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967330688:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767519.942395:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767519.942402:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767519/real 1775767519] req@ffff8f99703d1500 x1862022967330688/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767519.942410:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1500 x1862022967330688/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767519.942416:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767519.942418:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767519.942420:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967330688:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767519.942440:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967330816:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767519.942454:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767519.942462:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767519/real 1775767519] req@ffff8f99703d0700 x1862022967330816/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767519.942470:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0700 x1862022967330816/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767519.942476:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767519.942478:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767519.942481:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967330816:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767520.838013:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767520.838057:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767521.861822:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767521.861926:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767522.885851:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767522.885902:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767523.909844:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767523.909919:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767524.933883:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767524.933958:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767525.061842:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767525.061850:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767525.061854:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767525.061862:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4146 00000100:00080000:2.0:1775767525.061871:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767525.061876:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767525.061906:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3800 x1862022967330944/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767525.061929:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767525.061931:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767525.061933:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767525.061936:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4146 00000100:00080000:2.0:1775767525.061938:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767525.061941:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767525.061950:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767525.061958:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967330944:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767525.061960:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0380 x1862022967331072/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767525.061972:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767525.061974:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767525.061975:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767525.061977:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4146 00000100:00080000:2.0:1775767525.061980:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767525.061982:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767525.061985:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767525.061993:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0000 x1862022967331200/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767525.061999:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767525.062002:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767525.062003:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767525.062005:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4146 00000100:00080000:2.0:1775767525.062008:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767525.062011:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767525.062013:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775767525.062020:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767525.062023:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2680 x1862022967331328/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767525.062069:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767525/real 1775767525] req@ffff8f99703d3800 x1862022967330944/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767525.062082:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3800 x1862022967330944/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767525.062093:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767525.062097:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767525.062101:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967330944:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767525.062120:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967331072:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767525.062134:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767525.062141:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767525/real 1775767525] req@ffff8f99703d0380 x1862022967331072/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767525.062178:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0380 x1862022967331072/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767525.062183:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767525.062185:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767525.062188:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967331072:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767525.062192:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967331200:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767525.062204:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767525.062210:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767525/real 1775767525] req@ffff8f99703d0000 x1862022967331200/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767525.062217:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0000 x1862022967331200/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767525.062222:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767525.062224:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767525.062226:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967331200:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767525.062229:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967331328:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767525.062238:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767525.062243:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767525/real 1775767525] req@ffff8f99703d2680 x1862022967331328/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767525.062250:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2680 x1862022967331328/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767525.062270:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767525.062272:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767525.062275:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967331328:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767525.957881:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767525.957993:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767526.981863:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767526.981909:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767528.005911:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767528.005955:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775767529.029898:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767529.029959:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767530.053836:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767530.053892:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767530.181884:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767530.181892:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767530.181895:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767530.181904:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4151 00000100:00080000:0.0:1775767530.181914:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767530.181919:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767530.181950:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967331456/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767530.181972:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775767530.182004:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967331456:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775767530.182025:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767530.182027:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767530.182031:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4151 00000100:00080000:0.0:1775767530.182034:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767530.182037:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767530.182046:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767530.182057:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967331584/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775767530.182061:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775767530.182071:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767530.182074:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767530.182076:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767530.182079:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4151 00000100:00080000:0.0:1775767530.182082:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767530.182085:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767530.182088:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767530.182096:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967331712/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767530.182102:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767530.182107:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1775767530.182108:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767530/real 1775767530] req@ffff8f9970a75c00 x1862022967331456/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767530.182109:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767530.182115:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4151 00000100:00080000:0.0:1775767530.182119:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767530.182125:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767530.182139:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967331456/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1775767530.182147:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1775767530.182151:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767530.182155:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767530.182159:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967331456:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767530.182164:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967331840/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767530.182186:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967331584:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767530.182212:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767530.182225:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767530/real 1775767530] req@ffff8f9970a77100 x1862022967331584/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767530.182239:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967331584/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767530.182247:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767530.182249:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767530.182252:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967331584:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767530.182282:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967331712:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767530.182299:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767530.182307:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767530/real 1775767530] req@ffff8f9970a76a00 x1862022967331712/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767530.182315:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967331712/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767530.182320:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767530.182323:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767530.182325:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967331712:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767530.182329:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967331840:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767530.182339:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767530.182344:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767530/real 1775767530] req@ffff8f9970a74380 x1862022967331840/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767530.182351:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967331840/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767530.182356:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767530.182358:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767530.182359:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967331840:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767531.077861:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767531.077924:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767532.101844:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767532.101914:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767533.125855:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767533.125908:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767534.149863:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767534.149923:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767535.173868:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767535.173876:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00080000:2.0:1775767535.302430:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767535.302440:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767535.302445:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767535.302456:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4156 00000100:00080000:2.0:1775767535.302467:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767535.302475:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767535.302520:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2680 x1862022967331968/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767535.302550:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767535.302554:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767535.302556:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767535.302561:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4156 00000100:00080000:2.0:1775767535.302564:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767535.302568:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767535.302578:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767535.302593:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0000 x1862022967332096/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767535.302598:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967331968:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775767535.302624:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767535.302628:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767535.302630:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767535.302635:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4156 00000100:00080000:2.0:1775767535.302640:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767535.302644:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767535.302650:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767535.302663:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0380 x1862022967332224/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775767535.302674:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775767535.302677:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767535.302681:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767535.302683:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767535.302693:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4156 00000100:00080000:2.0:1775767535.302698:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767535.302714:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767535.302720:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767535.302733:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3800 x1862022967332352/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767535.302783:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767535/real 1775767535] req@ffff8f99703d2680 x1862022967331968/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767535.302825:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2680 x1862022967331968/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767535.302834:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767535.302838:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767535.302842:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967331968:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767535.302866:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967332096:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767535.302885:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767535.302894:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767535/real 1775767535] req@ffff8f99703d0000 x1862022967332096/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767535.302905:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0000 x1862022967332096/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767535.302914:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767535.302918:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767535.302921:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967332096:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767535.302927:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967332224:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767535.302948:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767535.302960:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767535/real 1775767535] req@ffff8f99703d0380 x1862022967332224/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767535.302973:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0380 x1862022967332224/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767535.303000:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767535.303003:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767535.303006:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967332224:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767535.303010:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967332352:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767535.303026:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767535.303034:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767535/real 1775767535] req@ffff8f99703d3800 x1862022967332352/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767535.303041:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3800 x1862022967332352/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767535.303047:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767535.303049:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767535.303051:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967332352:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767536.197844:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767536.197909:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767537.221802:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767537.221877:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767538.245827:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767538.245877:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767539.269817:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767539.269878:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767540.293847:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767540.293918:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767540.421846:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767540.421856:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767540.421861:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767540.421871:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4162 00000100:00080000:1.0:1775767540.421882:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767540.421890:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767540.421929:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967332480/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767540.421953:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767540.421955:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767540.421957:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767540.421960:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4162 00000100:00080000:1.0:1775767540.421963:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767540.421965:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767540.421973:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767540.421983:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967332608/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767540.421998:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967332480:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775767540.422019:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767540.422023:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767540.422026:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767540.422031:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4162 00000100:00080000:1.0:1775767540.422035:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767540.422039:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767540.422045:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775767540.422054:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767540.422062:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967332736/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767540.422071:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767540.422075:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767540.422076:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767540.422081:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4162 00000100:00080000:1.0:1775767540.422085:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767540.422089:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767540.422095:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767540.422102:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767540/real 1775767540] req@ffff8f994f2e4700 x1862022967332480/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767540.422111:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967332864/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767540.422122:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967332480/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767540.422132:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767540.422135:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767540.422139:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967332480:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767540.422175:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967332608:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767540.422192:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767540.422201:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767540/real 1775767540] req@ffff8f994f2e6300 x1862022967332608/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767540.422209:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967332608/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767540.422214:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767540.422217:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767540.422219:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967332608:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767540.422223:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967332736:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767540.422234:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767540.422239:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767540/real 1775767540] req@ffff8f994f2e4e00 x1862022967332736/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767540.422247:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967332736/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767540.422252:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767540.422253:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767540.422255:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967332736:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767540.422259:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967332864:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767540.422268:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767540.422274:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767540/real 1775767540] req@ffff8f994f2e7b80 x1862022967332864/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767540.422294:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967332864/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767540.422299:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767540.422301:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767540.422303:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967332864:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767541.317793:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767541.317866:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767542.341873:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767542.341885:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767543.365857:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767543.365922:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767544.389834:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767544.389902:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767545.413841:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767545.413887:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767545.541825:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767545.541834:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767545.541838:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767545.541847:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4167 00000100:00080000:1.0:1775767545.541857:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767545.541862:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767545.541896:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967332992/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767545.541920:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767545.541923:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767545.541928:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767545.541931:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4167 00000100:00080000:1.0:1775767545.541933:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767545.541936:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767545.541943:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767545.541952:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967333120/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767545.541959:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767545.541962:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767545.541963:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767545.541965:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4167 00000100:00080000:1.0:1775767545.541967:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767545.541972:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767545.541972:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967332992:192.168.202.146@tcp:250:kworker.0 00000008:00100000:1.0:1775767545.541975:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767545.541983:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967333248/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767545.542015:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767545.542017:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767545.542019:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767545.542022:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4167 00000100:00080000:1.0:1775767545.542024:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767545.542028:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:3.0:1775767545.542032:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:1.0:1775767545.542034:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767545.542045:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967333376/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767545.542069:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767545/real 1775767545] req@ffff8f994f2e7b80 x1862022967332992/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767545.542083:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967332992/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767545.542090:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767545.542093:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767545.542096:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967332992:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767545.542101:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967333120:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767545.542115:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767545.542123:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767545/real 1775767545] req@ffff8f994f2e4e00 x1862022967333120/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767545.542131:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967333120/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767545.542158:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767545.542161:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767545.542163:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967333120:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767545.542184:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967333248:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767545.542198:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767545.542205:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767545/real 1775767545] req@ffff8f994f2e6300 x1862022967333248/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767545.542213:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967333248/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767545.542219:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767545.542221:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767545.542223:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967333248:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767545.542226:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967333376:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767545.542236:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767545.542241:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767545/real 1775767545] req@ffff8f994f2e4700 x1862022967333376/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767545.542248:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967333376/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767545.542253:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767545.542255:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767545.542269:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967333376:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767546.437816:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767546.437894:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767547.461824:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767547.461872:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767548.485839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767548.485897:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767549.509888:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767549.509888:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775767550.533839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767550.533888:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767550.661828:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767550.661836:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767550.661841:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767550.661850:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4172 00000100:00080000:1.0:1775767550.661861:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767550.661867:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767550.661911:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967333504/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767550.661951:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767550.661956:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767550.661958:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775767550.661998:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967333504:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775767550.662001:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4172 00000100:00080000:1.0:1775767550.662008:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767550.662013:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767550.662021:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767550.662034:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967333632/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767550.662043:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767550.662046:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767550.662048:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767550.662051:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4172 00000100:00080000:1.0:1775767550.662054:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767550.662056:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767550.662059:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775767550.662060:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767550.662071:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967333760/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767550.662079:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767550.662082:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767550.662083:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767550.662085:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4172 00000100:00080000:1.0:1775767550.662087:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767550.662090:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767550.662093:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767550.662099:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967333888/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767550.662107:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767550/real 1775767550] req@ffff8f994f2e6680 x1862022967333504/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767550.662123:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967333504/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767550.662132:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767550.662136:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767550.662140:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967333504:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767550.662161:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967333632:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767550.662178:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767550.662190:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767550/real 1775767550] req@ffff8f994f2e4000 x1862022967333632/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767550.662198:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967333632/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767550.662203:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767550.662205:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767550.662209:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967333632:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767550.662214:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967333760:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767550.662229:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767550.662256:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767550/real 1775767550] req@ffff8f994f2e5f80 x1862022967333760/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767550.662267:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967333760/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767550.662275:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767550.662279:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767550.662282:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967333760:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767550.662288:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967333888:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767550.662302:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767550.662308:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767550/real 1775767550] req@ffff8f994f2e6a00 x1862022967333888/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767550.662317:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967333888/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767550.662322:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767550.662324:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767550.662328:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967333888:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767551.557822:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767551.557872:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767552.581821:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767552.581876:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767553.605788:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767553.605878:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767554.629816:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767554.629875:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767555.653848:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767555.653915:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767555.781829:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767555.781838:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767555.781842:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767555.781850:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4177 00000100:00080000:1.0:1775767555.781860:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767555.781866:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767555.781898:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967334016/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767555.781922:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767555.781926:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767555.781927:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767555.781932:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4177 00000100:00080000:1.0:1775767555.781935:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767555.781941:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767555.781950:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767555.781962:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967334144/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767555.781970:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967334016:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775767555.781971:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767555.781975:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767555.781977:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767555.781981:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4177 00000100:00080000:1.0:1775767555.781984:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767555.781987:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767555.781992:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767555.782000:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967334272/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767555.782008:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767555.782011:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767555.782013:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767555.782016:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4177 00000100:00080000:1.0:1775767555.782020:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767555.782023:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767555.782026:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775767555.782029:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767555.782035:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967334400/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767555.782065:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767555/real 1775767555] req@ffff8f994f2e6d80 x1862022967334016/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767555.782085:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967334016/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767555.782130:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767555.782133:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767555.782136:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967334016:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767555.782155:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967334144:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767555.782173:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767555.782181:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767555/real 1775767555] req@ffff8f994f2e4a80 x1862022967334144/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767555.782189:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967334144/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767555.782195:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767555.782197:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767555.782199:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967334144:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767555.782203:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967334272:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767555.782213:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767555.782219:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767555/real 1775767555] req@ffff8f994f2e7800 x1862022967334272/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767555.782226:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967334272/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767555.782231:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767555.782233:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767555.782234:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967334272:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767555.782250:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967334400:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767555.782262:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767555.782268:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767555/real 1775767555] req@ffff8f994f2e5180 x1862022967334400/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767555.782275:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967334400/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767555.782280:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767555.782281:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767555.782283:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967334400:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775767556.677907:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767556.677909:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775767557.701923:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767557.701975:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767558.725835:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767558.725890:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767559.749837:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767559.749891:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767560.773819:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767560.773886:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767560.901859:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767560.901868:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767560.901871:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767560.901886:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4182 00000100:00080000:0.0:1775767560.901896:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767560.901901:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767560.901934:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967334528/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767560.901957:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767560.901960:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767560.901961:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767560.901964:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4182 00000100:00080000:0.0:1775767560.901967:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767560.901969:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767560.901998:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967334528:192.168.202.146@tcp:250:kworker.0 00000008:00100000:0.0:1775767560.902017:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767560.902028:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967334656/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767560.902037:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767560.902040:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767560.902041:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767560.902044:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4182 00000100:00080000:0.0:1775767560.902052:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767560.902055:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:3.0:1775767560.902060:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:0.0:1775767560.902063:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767560.902073:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967334784/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767560.902080:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767560.902083:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767560.902084:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767560.902086:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4182 00000100:00080000:0.0:1775767560.902089:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767560.902091:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767560.902095:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767560.902102:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967334912/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767560.902112:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767560/real 1775767560] req@ffff8f9970a77800 x1862022967334528/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767560.902129:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967334528/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767560.902142:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767560.902146:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767560.902149:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967334528:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767560.902167:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967334656:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767560.902183:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767560.902191:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767560/real 1775767560] req@ffff8f9970a76d80 x1862022967334656/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767560.902217:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967334656/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767560.902223:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767560.902225:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767560.902227:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967334656:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767560.902231:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967334784:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767560.902244:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767560.902250:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767560/real 1775767560] req@ffff8f9970a74700 x1862022967334784/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767560.902258:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967334784/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767560.902263:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767560.902264:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767560.902266:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967334784:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767560.902270:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967334912:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767560.902279:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767560.902284:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767560/real 1775767560] req@ffff8f9970a74380 x1862022967334912/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767560.902291:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967334912/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767560.902311:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767560.902313:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767560.902315:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967334912:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767561.797840:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767561.797862:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767562.821799:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767562.821905:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767563.845918:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767563.845937:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767564.869877:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767564.869946:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767565.893864:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767565.893931:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767566.021890:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767566.021904:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767566.021911:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767566.021934:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4187 00000100:00080000:1.0:1775767566.021951:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767566.021961:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767566.022012:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967335040/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767566.022046:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767566.022050:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767566.022052:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767566.022058:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4187 00000100:00080000:1.0:1775767566.022063:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767566.022068:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767566.022079:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967335040:192.168.202.146@tcp:250:kworker.0 00000008:00100000:1.0:1775767566.022080:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767566.022100:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967335168/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767566.022116:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767566.022121:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767566.022123:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767566.022129:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4187 00000100:00080000:1.0:1775767566.022134:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767566.022140:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767566.022147:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775767566.022159:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767566.022162:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967335296/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767566.022174:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767566.022180:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767566.022182:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767566.022187:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4187 00000100:00080000:1.0:1775767566.022192:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767566.022196:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767566.022202:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767566.022207:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767566/real 1775767566] req@ffff8f994f2e5180 x1862022967335040/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767566.022219:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967335424/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767566.022225:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967335040/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767566.022242:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767566.022246:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767566.022249:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967335040:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767566.022285:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967335168:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767566.022301:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767566.022315:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767566/real 1775767566] req@ffff8f994f2e7800 x1862022967335168/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767566.022324:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967335168/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767566.022330:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767566.022332:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767566.022334:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967335168:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767566.022483:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967335296:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767566.022558:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767566.022590:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767566/real 1775767566] req@ffff8f994f2e4a80 x1862022967335296/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767566.022610:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967335296/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767566.022622:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767566.022627:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767566.022631:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967335296:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767566.022638:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967335424:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767566.022660:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767566.022669:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767566/real 1775767566] req@ffff8f994f2e6d80 x1862022967335424/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767566.022680:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967335424/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767566.022688:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767566.022691:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767566.022695:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967335424:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767566.917823:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767566.917936:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767567.941810:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767567.941885:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767568.965820:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767568.965881:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767569.989822:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767569.989876:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767571.013890:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767571.013939:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767571.141856:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767571.141867:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767571.141872:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767571.141887:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4192 00000100:00080000:1.0:1775767571.141899:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767571.141907:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767571.141952:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967335552/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767571.141981:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767571.141984:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767571.141985:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767571.141989:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4192 00000100:00080000:1.0:1775767571.141991:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767571.141994:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767571.142005:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767571.142014:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967335680/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767571.142021:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767571.142024:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767571.142025:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767571.142028:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4192 00000100:00080000:1.0:1775767571.142030:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767571.142032:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767571.142036:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767571.142042:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967335808/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767571.142049:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767571.142052:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767571.142053:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767571.142055:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4192 00000100:00080000:1.0:1775767571.142058:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767571.142062:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767571.142062:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967335552:192.168.202.146@tcp:250:kworker.0 00000008:00100000:1.0:1775767571.142069:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767571.142077:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967335936/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775767571.142142:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767571.142195:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767571/real 1775767571] req@ffff8f994f2e6a00 x1862022967335552/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767571.142331:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967335552/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767571.142342:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767571.142347:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767571.142352:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967335552:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767571.142393:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967335680:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767571.142452:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767571.142472:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767571/real 1775767571] req@ffff8f994f2e5f80 x1862022967335680/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767571.142481:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967335680/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767571.142487:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767571.142489:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767571.142492:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967335680:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767571.142517:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967335808:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767571.142531:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767571.142538:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767571/real 1775767571] req@ffff8f994f2e4000 x1862022967335808/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767571.142546:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967335808/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767571.142576:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767571.142579:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767571.142581:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967335808:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767571.142585:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967335936:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767571.142598:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767571.142605:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767571/real 1775767571] req@ffff8f994f2e6680 x1862022967335936/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767571.142613:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967335936/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767571.142618:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767571.142620:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767571.142622:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967335936:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767572.037819:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767572.037870:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767573.061808:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767573.061907:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767574.085821:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767574.085872:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767575.109877:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767575.109942:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767576.133821:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767576.133887:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767576.261860:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767576.261870:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767576.261876:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767576.261887:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4197 00000100:00080000:2.0:1775767576.261899:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767576.261907:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767576.261969:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0700 x1862022967336064/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767576.262003:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767576.262007:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767576.262009:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767576.262014:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4197 00000100:00080000:2.0:1775767576.262019:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767576.262024:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767576.262037:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767576.262054:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1500 x1862022967336192/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767576.262056:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967336064:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775767576.262067:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767576.262071:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767576.262072:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767576.262075:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4197 00000100:00080000:2.0:1775767576.262078:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767576.262080:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767576.262084:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767576.262093:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2d80 x1862022967336320/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767576.262100:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767576.262115:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767576.262116:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767576.262119:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4197 00000100:00080000:2.0:1775767576.262120:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767576.262123:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767576.262126:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767576.262132:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0e00 x1862022967336448/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767576.262236:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967336192:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767576.262266:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967336320:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767576.262279:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967336448:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767577.157886:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767577.157961:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767578.181900:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767578.181942:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775767579.205893:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767579.205994:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1775767579.334127:0:2367:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:0.0:1775767579.334185:0:2367:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1775767579.334212:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775767579.334266:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775767579.334295:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775767579.334310:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:1.0:1775767579.334312:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767576/real 1775767579] req@ffff8f99703d1500 x1862022967336192/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767579.334354:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1500 x1862022967336192/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767579.334373:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767579.334381:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767579.334386:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967336192:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767579.334397:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767576/real 1775767579] req@ffff8f99703d2d80 x1862022967336320/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767579.334413:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2d80 x1862022967336320/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767579.334422:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767579.334426:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767579.334430:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967336320:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767579.334438:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767576/real 1775767579] req@ffff8f99703d0e00 x1862022967336448/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767579.334451:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0e00 x1862022967336448/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767579.334460:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767579.334463:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767579.334467:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967336448:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767579.334535:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767576/real 1775767579] req@ffff8f99703d0700 x1862022967336064/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767579.334550:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0700 x1862022967336064/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767579.334560:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767579.334564:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767579.334569:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967336064:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767580.229827:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767580.229906:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767581.253811:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767581.253918:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767581.381854:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767581.381865:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767581.381870:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767581.381881:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4202 00000100:00080000:2.0:1775767581.381894:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767581.381906:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767581.381953:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3100 x1862022967336576/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767581.381983:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767581.381986:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767581.381988:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767581.381992:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4202 00000100:00080000:2.0:1775767581.381996:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767581.382000:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767581.382011:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767581.382024:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2300 x1862022967336704/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767581.382028:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967336576:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775767581.382041:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767581.382047:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767581.382049:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767581.382052:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4202 00000100:00080000:2.0:1775767581.382055:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767581.382059:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767581.382065:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767581.382078:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1180 x1862022967336832/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767581.382089:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767581.382094:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767581.382096:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767581.382100:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4202 00000100:00080000:2.0:1775767581.382105:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767581.382109:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767581.382114:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767581.382129:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3480 x1862022967336960/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767581.382139:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767581.382195:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767581/real 1775767581] req@ffff8f99703d3100 x1862022967336576/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767581.382217:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3100 x1862022967336576/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767581.382229:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767581.382235:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767581.382239:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967336576:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767581.382304:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967336704:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775767581.382332:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767581.382348:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767581/real 1775767581] req@ffff8f99703d2300 x1862022967336704/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767581.382371:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2300 x1862022967336704/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767581.382381:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767581.382385:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767581.382388:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967336704:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767581.382395:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967336832:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767581.382419:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767581.382432:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767581/real 1775767581] req@ffff8f99703d1180 x1862022967336832/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767581.382446:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1180 x1862022967336832/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767581.382455:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767581.382459:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767581.382463:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967336832:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767581.382469:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967336960:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767581.382491:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767581.382502:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767581/real 1775767581] req@ffff8f99703d3480 x1862022967336960/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767581.382514:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3480 x1862022967336960/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767581.382592:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767581.382597:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767581.382600:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967336960:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767582.277870:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767582.277931:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767583.301858:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767583.301921:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767584.325853:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767584.325945:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767585.349878:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767585.349918:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775767586.373850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767586.373914:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767586.501889:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767586.501899:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767586.501904:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767586.501915:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4208 00000100:00080000:1.0:1775767586.501928:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767586.501940:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767586.501985:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967337088/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767586.502021:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767586.502026:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767586.502028:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767586.502036:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4208 00000100:00080000:1.0:1775767586.502043:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767586.502048:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767586.502060:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767586.502068:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967337088:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767586.502088:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967337216/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767586.502111:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767586.502115:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767586.502117:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767586.502124:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4208 00000100:00080000:1.0:1775767586.502129:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767586.502134:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767586.502151:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767586.502168:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967337344/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767586.502180:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767586.502184:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767586.502186:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767586.502191:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4208 00000100:00080000:1.0:1775767586.502196:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767586.502200:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767586.502204:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767586.502245:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967337472/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775767586.502257:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767586.502441:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767586/real 1775767586] req@ffff8f994f2e4700 x1862022967337088/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767586.502623:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967337088/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767586.502632:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767586.502635:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767586.502639:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967337088:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767586.502662:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967337216:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767586.502708:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767586.502724:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767586/real 1775767586] req@ffff8f994f2e6300 x1862022967337216/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767586.502732:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967337216/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767586.502770:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767586.502773:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767586.502779:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967337216:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767586.502784:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967337344:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767586.502806:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767586.502818:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767586/real 1775767586] req@ffff8f994f2e4e00 x1862022967337344/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767586.502831:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967337344/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767586.502867:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767586.502870:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767586.502872:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967337344:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767586.502877:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967337472:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767586.502896:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767586.502907:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767586/real 1775767586] req@ffff8f994f2e7b80 x1862022967337472/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767586.502919:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967337472/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767586.502928:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767586.502932:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767586.502935:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967337472:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767587.397875:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767587.397958:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767588.421854:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767588.421938:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767589.445872:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767589.445940:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767590.469818:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767590.469940:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767591.493873:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767591.493936:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767591.621886:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767591.621897:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767591.621902:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767591.621912:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4213 00000100:00080000:0.0:1775767591.621926:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767591.621933:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767591.621980:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967337600/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767591.622009:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775767591.622049:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967337600:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775767591.622105:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767591.622109:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1775767591.622115:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775767591.622123:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4213 00000100:00080000:0.0:1775767591.622130:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767591.622136:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767591.622146:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767591.622175:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767591/real 1775767591] req@ffff8f9970a76a00 x1862022967337600/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767591.622187:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967337728/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767591.622196:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967337600/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767591.622202:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767591.622205:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767591.622207:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767591.622211:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4213 00000100:00080000:0.0:1775767591.622214:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767591.622217:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767591.622225:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1775767591.622230:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767591.622236:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767591.622238:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967337856/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767591.622246:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967337600:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775767591.622248:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767591.622250:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767591.622251:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767591.622257:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4213 00000100:00080000:0.0:1775767591.622258:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767591.622260:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767591.622281:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767591.622291:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967337984/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767591.622298:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967337728:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767591.622329:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767591.622339:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767591/real 1775767591] req@ffff8f9970a77100 x1862022967337728/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767591.622348:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967337728/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767591.622353:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767591.622356:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767591.622358:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967337728:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767591.622362:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967337856:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767591.622375:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767591.622381:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767591/real 1775767591] req@ffff8f9970a75c00 x1862022967337856/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767591.622388:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967337856/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767591.622393:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767591.622394:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767591.622396:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967337856:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767591.622406:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967337984:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767591.622416:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767591.622422:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767591/real 1775767591] req@ffff8f9970a77480 x1862022967337984/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767591.622442:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967337984/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767591.622448:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767591.622450:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767591.622452:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967337984:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767592.517912:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767592.517941:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767593.541880:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767593.541944:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767594.565844:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767594.565892:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767595.589839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767595.589905:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767596.613890:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767596.613976:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767596.741863:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767596.741878:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767596.741884:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767596.741898:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4218 00000100:00080000:0.0:1775767596.741914:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767596.741921:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767596.741973:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967338112/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767596.742002:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767596.742006:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767596.742007:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767596.742011:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4218 00000100:00080000:0.0:1775767596.742013:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767596.742017:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767596.742027:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767596.742037:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967338240/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767596.742045:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775767596.742048:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967338112:192.168.202.146@tcp:250:kworker.0 00000800:00000100:3.0:1775767596.742163:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775767596.742178:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767596.742183:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767596.742192:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4218 00000100:00080000:0.0:1775767596.742203:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767596.742210:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767596.742220:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767596.742236:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767596/real 1775767596] req@ffff8f9970a75180 x1862022967338112/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767596.742251:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967338368/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767596.742266:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967338112/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767596.742267:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767596.742278:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767596.742283:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767596.742299:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4218 00000100:00080000:0.0:1775767596.742307:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767596.742313:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00080000:3.0:1775767596.742313:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1775767596.742325:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1775767596.742330:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767596.742338:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967338112:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767596.742342:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967338496/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767596.742385:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967338240:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767596.742421:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767596.742436:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767596/real 1775767596] req@ffff8f9970a76300 x1862022967338240/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767596.742446:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967338240/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767596.742516:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767596.742521:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767596.742525:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967338240:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767596.742534:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967338368:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767596.742586:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767596.742608:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767596/real 1775767596] req@ffff8f9970a75500 x1862022967338368/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767596.742619:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967338368/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767596.742625:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767596.742627:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767596.742629:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967338368:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767596.742633:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967338496:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767596.742644:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767596.742650:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767596/real 1775767596] req@ffff8f9970a77480 x1862022967338496/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767596.742657:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967338496/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767596.742662:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767596.742664:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767596.742688:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967338496:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767597.637822:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767597.637924:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767598.661818:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767598.661888:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767599.685875:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767599.685910:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767600.709861:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767600.709931:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767601.733812:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767601.733855:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767601.861963:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767601.861975:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767601.862052:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767601.862091:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4223 00000100:00080000:1.0:1775767601.862106:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767601.862114:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767601.862160:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967338624/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767601.862195:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767601.862199:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767601.862202:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767601.862207:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4223 00000100:00080000:1.0:1775767601.862212:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775767601.862218:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967338624:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775767601.862220:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767601.862249:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767601.862267:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967338752/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767601.862294:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1775767601.862297:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775767601.862302:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767601.862304:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767601.862310:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4223 00000100:00080000:1.0:1775767601.862315:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767601.862320:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767601.862327:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767601.862341:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967338880/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767601.862348:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767601/real 1775767601] req@ffff8f994f2e7b80 x1862022967338624/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767601.862386:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767601.862390:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767601.862392:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775767601.862397:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967338624/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767601.862405:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4223 00000100:00080000:3.0:1775767601.862413:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767601.862415:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767601.862424:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00080000:3.0:1775767601.862424:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000008:00100000:1.0:1775767601.862434:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767601.862435:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967338624:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767601.862452:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967339008/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767601.862468:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967338752:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767601.862497:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767601.862512:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767601/real 1775767601] req@ffff8f994f2e4e00 x1862022967338752/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767601.862526:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967338752/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767601.862536:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767601.862539:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767601.862543:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967338752:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767601.862549:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967338880:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767601.862572:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767601.862606:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767601/real 1775767601] req@ffff8f994f2e6300 x1862022967338880/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767601.862629:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967338880/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767601.862639:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767601.862643:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767601.862647:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967338880:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767601.862653:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967339008:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767601.862675:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767601.862687:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767601/real 1775767601] req@ffff8f994f2e4700 x1862022967339008/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767601.862700:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967339008/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767601.862718:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767601.862722:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767601.862726:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967339008:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767602.757845:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767602.757897:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767603.781885:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767603.781986:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767604.805831:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767604.805916:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767605.829851:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767605.829911:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767606.853919:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767606.853954:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767606.981882:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767606.981897:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767606.982000:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767606.982023:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4228 00000100:00080000:0.0:1775767606.982034:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767606.982039:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767606.982069:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967339136/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767606.982093:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767606.982096:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767606.982097:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767606.982101:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4228 00000100:00080000:0.0:1775767606.982103:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767606.982106:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767606.982114:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767606.982124:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967339264/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767606.982128:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967339136:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775767606.982137:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767606.982140:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767606.982141:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767606.982144:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4228 00000100:00080000:0.0:1775767606.982146:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767606.982148:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767606.982151:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767606.982161:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967339392/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775767606.982195:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775767606.982208:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767606.982212:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767606.982214:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767606.982219:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4228 00000100:00080000:0.0:1775767606.982224:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767606.982227:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767606.982232:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767606.982236:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767606/real 1775767606] req@ffff8f9970a75c00 x1862022967339136/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767606.982245:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967339520/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767606.982258:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967339136/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767606.982288:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767606.982293:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767606.982298:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967339136:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767606.982335:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967339264:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767606.982362:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767606.982376:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767606/real 1775767606] req@ffff8f9970a77100 x1862022967339264/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767606.982390:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967339264/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767606.982399:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767606.982403:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767606.982405:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967339264:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767606.982410:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967339392:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767606.982427:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767606.982434:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767606/real 1775767606] req@ffff8f9970a76a00 x1862022967339392/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767606.982442:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967339392/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767606.982448:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767606.982450:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767606.982452:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967339392:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767606.982473:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967339520:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767606.982493:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767606.982504:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767606/real 1775767606] req@ffff8f9970a74380 x1862022967339520/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767606.982517:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967339520/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767606.982525:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767606.982529:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767606.982533:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967339520:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767607.877836:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767607.877897:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767608.901879:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767608.901926:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767609.925832:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767609.925887:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767610.949842:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767610.949902:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767611.973817:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767611.973865:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767612.101853:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767612.101867:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767612.101871:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767612.101879:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4233 00000100:00080000:0.0:1775767612.101889:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767612.101895:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767612.101928:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967339648/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767612.101952:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767612.101954:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767612.101956:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767612.101973:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4233 00000100:00100000:3.0:1775767612.101999:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967339648:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775767612.102012:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767612.102017:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767612.102025:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767612.102038:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967339776/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767612.102047:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767612.102049:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767612.102051:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767612.102053:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4233 00000800:00000100:3.0:1775767612.102058:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775767612.102059:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767612.102064:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767612.102068:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767612.102086:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967339904/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767612.102093:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775767612.102094:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767612/real 1775767612] req@ffff8f9970a74700 x1862022967339648/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767612.102098:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767612.102101:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767612.102111:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4233 00000100:00080000:0.0:1775767612.102118:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767612.102122:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767612.102126:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767612.102128:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967339648/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767612.102137:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967340032/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767612.102137:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767612.102145:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767612.102152:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967339648:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767612.102173:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967339776:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767612.102195:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767612.102208:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767612/real 1775767612] req@ffff8f9970a76d80 x1862022967339776/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767612.102246:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967339776/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767612.102255:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767612.102259:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767612.102263:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967339776:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767612.102269:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967339904:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767612.102286:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767612.102295:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767612/real 1775767612] req@ffff8f9970a77800 x1862022967339904/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767612.102303:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967339904/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767612.102308:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767612.102311:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767612.102313:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967339904:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767612.102316:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967340032:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767612.102327:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767612.102332:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767612/real 1775767612] req@ffff8f9970a74380 x1862022967340032/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767612.102339:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967340032/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767612.102362:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767612.102364:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767612.102367:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967340032:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767612.997841:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767612.997979:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767614.021960:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767614.021970:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767615.045850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767615.045943:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767616.069836:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767616.069903:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767617.093822:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767617.093940:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767617.221893:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767617.221905:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767617.221925:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767617.221935:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4238 00000100:00080000:0.0:1775767617.221948:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767617.221963:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767617.221997:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967340160/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767617.222020:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767617.222022:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767617.222024:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767617.222027:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4238 00000100:00080000:0.0:1775767617.222031:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767617.222034:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767617.222045:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767617.222054:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967340288/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767617.222065:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967340160:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775767617.222067:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767617.222082:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767617.222083:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767617.222087:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4238 00000100:00080000:0.0:1775767617.222111:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767617.222114:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767617.222119:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767617.222128:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967340416/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767617.222135:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767617.222138:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767617.222139:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767617.222141:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4238 00000100:00080000:0.0:1775767617.222143:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767617.222147:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767617.222150:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767617.222156:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967340544/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775767617.222166:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767617.222234:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767617/real 1775767617] req@ffff8f9970a76a00 x1862022967340160/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767617.222256:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967340160/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767617.222266:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767617.222271:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767617.222275:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967340160:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767617.222304:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967340288:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767617.222323:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767617.222332:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767617/real 1775767617] req@ffff8f9970a77100 x1862022967340288/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767617.222340:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967340288/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767617.222346:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767617.222349:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767617.222351:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967340288:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767617.222463:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967340416:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767617.222526:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767617.222552:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767617/real 1775767617] req@ffff8f9970a75c00 x1862022967340416/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767617.222570:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967340416/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767617.222591:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767617.222596:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767617.222601:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967340416:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767617.222607:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967340544:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767617.222629:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767617.222641:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767617/real 1775767617] req@ffff8f9970a77480 x1862022967340544/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767617.222654:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967340544/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767617.222661:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767617.222665:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767617.222668:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967340544:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767618.117854:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767618.117967:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767619.141815:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767619.141959:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767620.165925:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767620.166008:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767621.189883:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767621.189959:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767622.213853:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767622.213940:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767622.341814:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767622.341822:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767622.341826:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767622.341834:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4243 00000100:00080000:2.0:1775767622.341844:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767622.341850:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767622.341897:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3b80 x1862022967340672/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767622.341923:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767622.341926:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767622.341927:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767622.341931:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4243 00000100:00080000:2.0:1775767622.341934:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767622.341938:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767622.341947:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767622.341954:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967340672:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767622.341959:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1880 x1862022967340800/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767622.341969:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767622.341971:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767622.341973:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767622.341976:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4243 00000100:00080000:2.0:1775767622.341978:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767622.341982:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767622.341986:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767622.341996:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2a00 x1862022967340928/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767622.342003:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767622.342006:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767622.342008:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767622.342015:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4243 00000100:00080000:2.0:1775767622.342019:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767622.342021:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:0.0:1775767622.342024:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:2.0:1775767622.342026:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767622.342034:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1c00 x1862022967341056/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767622.342062:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767622/real 1775767622] req@ffff8f99703d3b80 x1862022967340672/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767622.342077:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3b80 x1862022967340672/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767622.342085:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767622.342121:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767622.342124:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967340672:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767622.342143:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967340800:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775767622.342160:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767622.342168:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767622/real 1775767622] req@ffff8f99703d1880 x1862022967340800/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767622.342177:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1880 x1862022967340800/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767622.342182:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767622.342191:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767622.342194:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967340800:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767622.342198:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967340928:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767622.342212:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767622.342220:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767622/real 1775767622] req@ffff8f99703d2a00 x1862022967340928/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767622.342228:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2a00 x1862022967340928/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767622.342233:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767622.342235:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767622.342238:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967340928:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767622.342241:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967341056:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767622.342253:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767622.342259:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767622/real 1775767622] req@ffff8f99703d1c00 x1862022967341056/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767622.342267:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1c00 x1862022967341056/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767622.342273:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767622.342274:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767622.342276:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967341056:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767623.237825:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767623.237931:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767624.261859:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767624.261891:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767625.285853:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767625.285913:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767626.309864:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767626.309939:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767627.333865:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767627.333928:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767627.461883:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767627.461893:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767627.461898:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767627.461908:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4249 00000100:00080000:0.0:1775767627.461918:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767627.461924:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767627.461955:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967341184/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767627.461980:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767627.461984:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767627.461986:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767627.461991:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4249 00000100:00080000:0.0:1775767627.461994:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767627.461998:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767627.462019:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967341184:192.168.202.146@tcp:250:kworker.0 00000008:00100000:0.0:1775767627.462042:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767627.462054:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967341312/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767627.462065:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767627.462068:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767627.462070:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767627.462073:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4249 00000100:00080000:0.0:1775767627.462076:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767627.462079:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:3.0:1775767627.462080:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:0.0:1775767627.462084:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767627.462100:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967341440/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767627.462110:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767627.462113:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767627.462115:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767627.462119:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4249 00000100:00080000:0.0:1775767627.462122:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775767627.462125:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767627/real 1775767627] req@ffff8f9970a75500 x1862022967341184/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767627.462126:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767627.462130:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767627.462142:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967341568/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767627.462161:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967341184/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767627.462171:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767627.462176:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767627.462180:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967341184:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767627.462204:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967341312:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767627.462223:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767627.462232:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767627/real 1775767627] req@ffff8f9970a76300 x1862022967341312/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767627.462241:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967341312/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767627.462246:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767627.462248:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767627.462250:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967341312:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767627.462254:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967341440:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767627.462271:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767627.462281:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767627/real 1775767627] req@ffff8f9970a75180 x1862022967341440/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767627.462293:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967341440/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767627.462326:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767627.462330:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767627.462333:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967341440:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767627.462340:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967341568:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767627.462358:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767627.462366:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767627/real 1775767627] req@ffff8f9970a77480 x1862022967341568/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767627.462373:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967341568/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767627.462379:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767627.462381:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767627.462383:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967341568:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767628.357938:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767628.357967:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767629.381874:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767629.381920:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767630.405853:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767630.405909:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767631.429817:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767631.429885:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767632.453850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767632.453906:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767632.581860:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767632.581867:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767632.581872:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767632.581880:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4254 00000100:00080000:2.0:1775767632.581889:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767632.581895:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767632.581927:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1c00 x1862022967341696/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767632.581948:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767632.581951:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767632.581953:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767632.581956:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4254 00000100:00080000:2.0:1775767632.581958:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767632.581961:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767632.581968:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767632.581980:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967341696:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767632.581981:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2a00 x1862022967341824/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767632.581994:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767632.581996:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767632.581997:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767632.582000:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4254 00000100:00080000:2.0:1775767632.582003:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767632.582006:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767632.582009:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767632.582017:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1880 x1862022967341952/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767632.582023:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767632.582026:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767632.582027:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767632.582029:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4254 00000100:00080000:2.0:1775767632.582033:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767632.582035:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767632.582037:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775767632.582044:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767632.582046:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3b80 x1862022967342080/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767632.582099:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767632/real 1775767632] req@ffff8f99703d1c00 x1862022967341696/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767632.582115:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1c00 x1862022967341696/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767632.582123:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767632.582126:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767632.582129:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967341696:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767632.582174:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967341824:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767632.582190:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767632.582199:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767632/real 1775767632] req@ffff8f99703d2a00 x1862022967341824/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767632.582207:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2a00 x1862022967341824/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767632.582212:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767632.582214:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767632.582217:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967341824:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767632.582220:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967341952:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767632.582231:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767632.582236:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767632/real 1775767632] req@ffff8f99703d1880 x1862022967341952/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767632.582243:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1880 x1862022967341952/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767632.582248:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767632.582250:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767632.582252:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967341952:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767632.582255:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967342080:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767632.582265:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767632.582270:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767632/real 1775767632] req@ffff8f99703d3b80 x1862022967342080/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767632.582289:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3b80 x1862022967342080/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767632.582295:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767632.582297:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767632.582299:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967342080:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767633.477857:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767633.477920:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767634.501846:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767634.501891:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767635.525911:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767635.526172:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767636.549831:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767636.549856:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767637.573879:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767637.574080:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767637.701890:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767637.701904:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767637.701911:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767637.701924:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4259 00000100:00080000:3.0:1775767637.701937:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767637.701946:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767637.702004:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddc00 x1862022967342208/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767637.702041:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767637.702046:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767637.702048:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767637.702054:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4259 00000100:00080000:3.0:1775767637.702058:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767637.702070:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767637.702083:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767637.702105:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967342336/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767637.702108:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967342208:192.168.202.146@tcp:250:kworker.0 00000800:00000100:0.0:1775767637.702198:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775767637.702253:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767637.702260:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1775767637.702262:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767637/real 1775767637] req@ffff8f9970eddc00 x1862022967342208/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767637.702264:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767637.702275:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4259 00000100:00080000:3.0:1775767637.702288:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767637.702294:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767637.702303:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddc00 x1862022967342208/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1775767637.702305:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1775767637.702327:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767637.702334:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767637.702341:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967342208:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767637.702343:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967342464/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767637.702364:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767637.702367:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767637.702369:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767637.702372:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4259 00000100:00080000:3.0:1775767637.702374:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767637.702376:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767637.702379:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767637.702387:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf100 x1862022967342592/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767637.702395:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967342336:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775767637.702430:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767637.702453:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767637/real 1775767637] req@ffff8f9970edce00 x1862022967342336/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767637.702469:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967342336/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767637.702479:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767637.702483:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767637.702487:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967342336:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767637.702495:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967342464:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767637.702519:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767637.702531:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767637/real 1775767637] req@ffff8f9970edd500 x1862022967342464/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767637.702654:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967342464/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767637.702663:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767637.702666:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767637.702669:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967342464:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767637.702695:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967342592:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767637.702796:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767637.702813:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767637/real 1775767637] req@ffff8f9970edf100 x1862022967342592/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767637.702822:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf100 x1862022967342592/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767637.702828:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767637.702831:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767637.702833:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967342592:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767638.597834:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767638.597869:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767639.621849:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767639.621912:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767640.645810:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767640.645862:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767641.669827:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767641.669889:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767642.693857:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767642.693871:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00080000:3.0:1775767642.821844:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767642.821853:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767642.821858:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767642.821866:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4264 00000100:00080000:3.0:1775767642.821899:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767642.821906:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767642.821938:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967342720/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767642.821958:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767642.821961:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767642.821963:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767642.821971:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4264 00000100:00080000:3.0:1775767642.821974:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767642.821979:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767642.821987:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767642.821999:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967342848/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767642.822008:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967342720:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775767642.822012:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767642.822017:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767642.822019:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767642.822024:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4264 00000100:00080000:3.0:1775767642.822029:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767642.822033:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767642.822039:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767642.822054:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967342976/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767642.822064:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767642.822079:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767642.822081:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767642.822084:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4264 00000100:00080000:3.0:1775767642.822087:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767642.822090:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767642.822094:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767642.822105:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967343104/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767642.822112:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967342848:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767642.822141:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967342976:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767642.822157:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967343104:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767643.717839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767643.717892:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767644.741842:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767644.741912:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767645.765870:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767645.765976:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775767645.894024:0:2369:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:1.0:1775767645.894072:0:2369:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775767645.894090:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775767645.894134:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775767645.894144:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775767645.894152:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:0.0:1775767645.894194:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767642/real 1775767645] req@ffff8f9970edc000 x1862022967343104/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767645.894224:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967343104/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767645.894236:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767645.894242:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767645.894246:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967343104:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767645.894254:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767642/real 1775767645] req@ffff8f9970edc700 x1862022967342848/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767645.894267:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967342848/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767645.894275:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767645.894279:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767645.894282:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967342848:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767645.894288:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767642/real 1775767645] req@ffff8f9970ede680 x1862022967342976/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767645.894299:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967342976/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767645.894305:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767645.894308:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767645.894338:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967342976:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767645.894346:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767642/real 1775767645] req@ffff8f9970eded80 x1862022967342720/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767645.894360:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967342720/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767645.894368:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767645.894372:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767645.894375:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967342720:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767646.789841:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767646.789915:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767647.813841:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767647.813923:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767647.941884:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767647.941896:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767647.941902:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767647.941916:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4269 00000100:00080000:3.0:1775767647.941929:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767647.941936:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767647.941985:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967343232/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767647.942015:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767647.942020:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767647.942022:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767647.942026:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4269 00000100:00080000:3.0:1775767647.942030:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767647.942035:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767647.942048:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767647.942091:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967343360/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767647.942125:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967343232:192.168.202.146@tcp:250:kworker.0 00000800:00000100:0.0:1775767647.942207:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775767647.942236:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767647.942241:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767647.942244:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767647.942252:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4269 00000100:00100000:0.0:1775767647.942259:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767647/real 1775767647] req@ffff8f9970edf800 x1862022967343232/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767647.942259:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767647.942270:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767647.942287:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767647.942291:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967343232/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767647.942316:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1775767647.942320:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967343488/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767647.942325:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767647.942335:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967343232:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775767647.942342:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767647.942346:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767647.942348:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767647.942360:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4269 00000100:00080000:3.0:1775767647.942365:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775767647.942371:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967343360:192.168.202.146@tcp:38:kworker.0 00000100:00080000:3.0:1775767647.942372:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767647.942382:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767647.942400:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967343616/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767647.942409:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767647.942431:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767647/real 1775767647] req@ffff8f9970edf480 x1862022967343360/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767647.942450:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967343360/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767647.942459:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767647.942462:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767647.942466:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967343360:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767647.942472:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967343488:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767647.942491:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767647.942501:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767647/real 1775767647] req@ffff8f9970edd880 x1862022967343488/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767647.942512:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967343488/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767647.942521:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767647.942524:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767647.942528:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967343488:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767647.942561:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967343616:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767647.942582:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767647.942593:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767647/real 1775767647] req@ffff8f9970ede300 x1862022967343616/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767647.942605:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967343616/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767647.942658:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767647.942662:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767647.942665:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967343616:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767648.837815:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767648.837861:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767649.861886:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767649.861892:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767650.885824:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767650.885871:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767651.909834:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767651.909898:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767652.933848:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767652.933902:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767653.061862:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767653.061873:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767653.061878:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767653.061888:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4274 00000100:00080000:1.0:1775767653.061898:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767653.061903:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767653.061937:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967343744/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767653.061965:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767653.061968:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767653.061970:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767653.061973:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4274 00000100:00080000:1.0:1775767653.061975:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767653.061977:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767653.061984:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767653.061993:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967343872/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767653.062000:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767653.062002:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767653.062003:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767653.062006:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4274 00000100:00080000:1.0:1775767653.062008:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767653.062010:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767653.062013:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767653.062020:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967343744:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767653.062021:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967344000/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767653.062033:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767653.062035:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767653.062036:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767653.062039:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4274 00000100:00080000:1.0:1775767653.062077:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775767653.062080:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775767653.062083:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767653.062087:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767653.062097:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967344128/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767653.062120:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767653/real 1775767653] req@ffff8f994f2e6680 x1862022967343744/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767653.062134:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967343744/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767653.062141:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767653.062144:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767653.062147:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967343744:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767653.062152:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967343872:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775767653.062171:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767653.062183:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767653/real 1775767653] req@ffff8f994f2e4000 x1862022967343872/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767653.062196:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967343872/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767653.062204:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767653.062209:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767653.062212:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967343872:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767653.062249:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967344000:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767653.062272:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767653.062283:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767653/real 1775767653] req@ffff8f994f2e5f80 x1862022967344000/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767653.062291:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967344000/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767653.062297:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767653.062299:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767653.062301:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967344000:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767653.062305:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967344128:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767653.062317:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767653.062323:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767653/real 1775767653] req@ffff8f994f2e6a00 x1862022967344128/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767653.062330:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967344128/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767653.062335:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767653.062336:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767653.062338:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967344128:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767653.957854:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767653.957927:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767654.981851:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767654.981927:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767656.005850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767656.005904:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767657.029894:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767657.029959:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767658.053813:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767658.053880:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767658.181866:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767658.181875:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767658.181880:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767658.181890:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4279 00000100:00080000:1.0:1775767658.181900:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767658.181906:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767658.181937:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967344256/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767658.181958:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767658.181961:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767658.181962:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767658.181965:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4279 00000100:00080000:1.0:1775767658.181968:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767658.181970:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767658.181978:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767658.181989:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967344384/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767658.181994:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967344256:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775767658.182028:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767658.182032:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767658.182034:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767658.182040:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4279 00000100:00080000:1.0:1775767658.182045:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767658.182051:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:0.0:1775767658.182053:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:1.0:1775767658.182062:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767658.182079:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967344512/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767658.182090:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767658/real 1775767658] req@ffff8f994f2e6d80 x1862022967344256/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767658.182091:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767658.182102:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767658.182106:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767658.182112:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4279 00000100:00100000:0.0:1775767658.182115:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967344256/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767658.182118:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767658.182127:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00080000:0.0:1775767658.182128:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767658.182135:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767658.182138:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967344256:192.168.202.146@tcp:250:kworker.0 00000008:00100000:1.0:1775767658.182150:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767658.182158:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967344384:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767658.182167:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967344640/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767658.182187:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767658.182206:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767658/real 1775767658] req@ffff8f994f2e4a80 x1862022967344384/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767658.182221:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967344384/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767658.182229:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767658.182232:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767658.182234:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967344384:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767658.182239:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967344512:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767658.182255:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767658.182263:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767658/real 1775767658] req@ffff8f994f2e7800 x1862022967344512/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767658.182291:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967344512/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767658.182297:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767658.182300:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767658.182302:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967344512:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767658.182314:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967344640:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767658.182328:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767658.182336:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767658/real 1775767658] req@ffff8f994f2e5180 x1862022967344640/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767658.182344:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967344640/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767658.182350:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767658.182353:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767658.182356:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967344640:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767659.077881:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767659.077937:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767660.101822:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767660.101863:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767661.125869:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767661.125940:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767662.149853:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767662.149908:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767663.173859:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767663.173922:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767663.301837:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767663.301848:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767663.301893:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767663.301904:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4284 00000100:00080000:1.0:1775767663.301916:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767663.301923:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767663.301962:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967344768/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767663.301992:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767663.301995:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767663.301997:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767663.302001:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4284 00000100:00080000:1.0:1775767663.302005:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767663.302009:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767663.302019:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767663.302033:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967344768:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767663.302036:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967344896/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767663.302050:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767663.302053:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767663.302055:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767663.302059:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4284 00000100:00080000:1.0:1775767663.302062:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767663.302067:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767663.302072:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767663.302083:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967345024/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767663.302095:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775767663.302106:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767663.302110:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767663.302112:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767663.302117:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4284 00000100:00080000:1.0:1775767663.302121:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767663.302126:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767663.302131:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767663.302133:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767663/real 1775767663] req@ffff8f994f2e5180 x1862022967344768/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767663.302148:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967345152/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767663.302158:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967344768/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767663.302167:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767663.302171:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767663.302174:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967344768:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767663.302194:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967344896:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775767663.302208:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767663.302216:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767663/real 1775767663] req@ffff8f994f2e7800 x1862022967344896/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767663.302223:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967344896/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767663.302229:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767663.302231:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767663.302233:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967344896:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767663.302237:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967345024:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767663.302246:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767663.302252:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767663/real 1775767663] req@ffff8f994f2e4a80 x1862022967345024/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767663.302258:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967345024/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767663.302268:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767663.302270:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767663.302272:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967345024:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767663.302275:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967345152:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767663.302284:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767663.302289:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767663/real 1775767663] req@ffff8f994f2e6d80 x1862022967345152/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767663.302296:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967345152/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767663.302301:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767663.302303:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767663.302304:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967345152:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767664.197889:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767664.197900:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767665.221881:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767665.221954:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767666.245882:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767666.245979:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767667.269831:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767667.269931:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767668.293815:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767668.293925:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767668.422367:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767668.422378:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767668.422382:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767668.422393:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4290 00000100:00080000:2.0:1775767668.422404:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767668.422412:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767668.422483:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3480 x1862022967345280/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767668.422512:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767668.422516:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767668.422518:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767668.422546:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4290 00000100:00080000:2.0:1775767668.422551:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767668.422556:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767668.422567:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767668.422578:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967345280:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767668.422596:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1180 x1862022967345408/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767668.422608:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767668.422612:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767668.422614:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767668.422617:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4290 00000100:00080000:2.0:1775767668.422621:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767668.422626:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767668.422632:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767668.422659:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2300 x1862022967345536/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767668.422669:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775767668.422672:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767668.422676:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767668.422678:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767668.422683:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4290 00000100:00080000:2.0:1775767668.422694:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767668.422699:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767668.422705:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767668.422712:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767668/real 1775767668] req@ffff8f99703d3480 x1862022967345280/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767668.422729:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3480 x1862022967345280/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767668.422735:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3100 x1862022967345664/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767668.422763:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767668.422771:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767668.422782:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967345280:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767668.422813:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967345408:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775767668.422838:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767668.422869:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767668/real 1775767668] req@ffff8f99703d1180 x1862022967345408/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767668.422883:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1180 x1862022967345408/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767668.422889:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767668.422891:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767668.422894:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967345408:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767668.422901:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967345536:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767668.422919:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767668.422927:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767668/real 1775767668] req@ffff8f99703d2300 x1862022967345536/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767668.422940:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2300 x1862022967345536/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767668.422948:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767668.422952:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767668.422955:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967345536:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767668.422962:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967345664:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767668.422982:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767668.422992:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767668/real 1775767668] req@ffff8f99703d3100 x1862022967345664/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767668.423000:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3100 x1862022967345664/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767668.423023:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767668.423027:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767668.423030:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967345664:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767669.317826:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767669.317873:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767670.341808:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767670.341869:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767671.365916:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767671.366183:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767672.389858:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767672.389925:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767673.413834:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767673.413894:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767673.541801:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767673.541810:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767673.541814:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767673.541822:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4295 00000100:00080000:2.0:1775767673.541833:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767673.541875:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767673.541908:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0700 x1862022967345792/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767673.541931:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767673.541934:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767673.541935:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767673.541938:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4295 00000100:00080000:2.0:1775767673.541941:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767673.541943:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767673.541950:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767673.541958:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0e00 x1862022967345920/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767673.541966:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767673.541968:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767673.541969:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767673.541972:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4295 00000100:00080000:2.0:1775767673.541974:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767673.541976:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767673.541980:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767673.541988:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2d80 x1862022967346048/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767673.541989:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967345792:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775767673.542001:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767673.542004:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767673.542005:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767673.542008:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4295 00000100:00080000:2.0:1775767673.542009:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767673.542011:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767673.542013:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767673.542037:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1500 x1862022967346176/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767673.542064:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767673.542106:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767673/real 1775767673] req@ffff8f99703d0700 x1862022967345792/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767673.542126:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0700 x1862022967345792/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767673.542138:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767673.542143:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767673.542147:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967345792:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767673.542155:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967345920:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775767673.542178:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767673.542191:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767673/real 1775767673] req@ffff8f99703d0e00 x1862022967345920/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767673.542205:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0e00 x1862022967345920/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767673.542213:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767673.542237:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767673.542241:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967345920:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767673.542265:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967346048:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767673.542287:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767673.542299:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767673/real 1775767673] req@ffff8f99703d2d80 x1862022967346048/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767673.542312:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2d80 x1862022967346048/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767673.542320:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767673.542324:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767673.542327:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967346048:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767673.542334:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967346176:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767673.542354:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767673.542382:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767673/real 1775767673] req@ffff8f99703d1500 x1862022967346176/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767673.542397:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1500 x1862022967346176/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767673.542405:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767673.542409:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767673.542412:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967346176:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775767674.437927:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767674.437985:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775767675.461815:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767675.461869:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767676.485842:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767676.485953:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767677.509832:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767677.509949:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767678.533883:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767678.533910:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767678.661817:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767678.661825:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767678.661829:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767678.661838:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4300 00000100:00080000:2.0:1775767678.661848:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767678.661853:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767678.661885:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3800 x1862022967346304/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767678.661907:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767678.661910:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767678.661911:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767678.661914:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4300 00000100:00080000:2.0:1775767678.661916:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767678.661919:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767678.661926:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767678.661934:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0380 x1862022967346432/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767678.661941:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767678.661944:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767678.661945:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767678.661947:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4300 00000100:00080000:2.0:1775767678.661949:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767678.661951:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767678.661954:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767678.661959:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0000 x1862022967346560/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767678.661966:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767678.661968:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767678.661970:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767678.661972:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4300 00000100:00080000:2.0:1775767678.661974:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767678.661976:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767678.661978:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767678.661984:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2680 x1862022967346688/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767678.661993:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967346304:192.168.202.146@tcp:250:kworker.0 00000800:00000100:1.0:1775767678.662097:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767678.662140:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767678/real 1775767678] req@ffff8f99703d3800 x1862022967346304/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767678.662153:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3800 x1862022967346304/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767678.662162:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767678.662166:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767678.662169:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967346304:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767678.662177:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967346432:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775767678.662192:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767678.662200:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767678/real 1775767678] req@ffff8f99703d0380 x1862022967346432/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767678.662207:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0380 x1862022967346432/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767678.662212:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767678.662214:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767678.662216:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967346432:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767678.662219:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967346560:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767678.662243:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767678.662249:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767678/real 1775767678] req@ffff8f99703d0000 x1862022967346560/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767678.662258:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0000 x1862022967346560/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767678.662263:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767678.662265:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767678.662267:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967346560:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767678.662283:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967346688:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767678.662294:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767678.662300:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767678/real 1775767678] req@ffff8f99703d2680 x1862022967346688/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767678.662307:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2680 x1862022967346688/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767678.662312:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767678.662314:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767678.662340:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967346688:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767679.557823:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767679.557848:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767680.581886:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767680.581979:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767681.605846:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767681.605928:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767682.629846:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767682.629927:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767683.653823:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767683.653878:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767683.781910:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767683.781918:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767683.781921:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767683.781930:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4305 00000100:00080000:0.0:1775767683.781940:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767683.781945:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767683.782022:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967346816/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767683.782045:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767683.782048:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767683.782049:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767683.782052:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4305 00000100:00080000:0.0:1775767683.782055:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767683.782058:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767683.782066:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767683.782084:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967346944/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767683.782092:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775767683.782096:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967346816:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775767683.782097:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767683.782099:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767683.782106:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4305 00000100:00080000:0.0:1775767683.782109:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767683.782111:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767683.782115:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767683.782132:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967347072/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767683.782139:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767683.782142:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767683.782143:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767683.782145:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4305 00000100:00080000:0.0:1775767683.782154:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767683.782157:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767683.782160:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775767683.782165:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767683.782179:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967347200/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767683.782210:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767683/real 1775767683] req@ffff8f9970a75c00 x1862022967346816/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767683.782230:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967346816/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767683.782241:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767683.782246:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767683.782250:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967346816:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767683.782288:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967346944:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775767683.782310:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767683.782321:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767683/real 1775767683] req@ffff8f9970a77100 x1862022967346944/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767683.782330:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967346944/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767683.782336:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767683.782338:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767683.782340:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967346944:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767683.782345:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967347072:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767683.782356:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767683.782362:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767683/real 1775767683] req@ffff8f9970a76a00 x1862022967347072/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767683.782370:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967347072/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767683.782384:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767683.782386:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767683.782388:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967347072:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767683.782393:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967347200:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767683.782404:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767683.782410:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767683/real 1775767683] req@ffff8f9970a74380 x1862022967347200/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767683.782418:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967347200/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767683.782423:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767683.782425:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767683.782427:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967347200:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767684.677856:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767684.677927:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767685.701867:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767685.701892:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767686.725859:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767686.725951:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767687.749816:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767687.749909:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767688.773850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767688.773924:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767688.901819:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767688.901827:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767688.901831:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767688.901840:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4310 00000100:00080000:0.0:1775767688.901850:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767688.901855:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767688.901887:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967347328/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767688.901911:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767688.901914:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767688.901937:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767688.901941:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4310 00000100:00080000:0.0:1775767688.901944:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767688.901947:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767688.901955:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767688.901956:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967347328:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767688.901967:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967347456/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767688.901977:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767688.901979:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767688.901980:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767688.901983:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4310 00000100:00080000:0.0:1775767688.901986:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767688.901988:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767688.901992:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767688.901999:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967347584/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767688.902007:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767688.902009:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767688.902010:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767688.902012:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4310 00000100:00080000:0.0:1775767688.902015:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1775767688.902017:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775767688.902020:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767688.902024:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767688.902044:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967347712/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767688.902057:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767688/real 1775767688] req@ffff8f9970a77800 x1862022967347328/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767688.902072:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967347328/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767688.902079:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767688.902082:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767688.902085:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967347328:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767688.902101:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967347456:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775767688.902149:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767688.902163:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767688/real 1775767688] req@ffff8f9970a76d80 x1862022967347456/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767688.902180:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967347456/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767688.902186:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767688.902188:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767688.902190:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967347456:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767688.902195:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967347584:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767688.902209:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767688.902238:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767688/real 1775767688] req@ffff8f9970a74700 x1862022967347584/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767688.902248:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967347584/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767688.902254:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767688.902256:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767688.902258:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967347584:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767688.902262:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967347712:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767688.902276:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767688.902282:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767688/real 1775767688] req@ffff8f9970a74380 x1862022967347712/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767688.902295:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967347712/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767688.902300:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767688.902302:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767688.902303:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967347712:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767689.797839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767689.797868:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767690.821866:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767690.821879:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767691.845795:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767691.845871:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767692.869891:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767692.869899:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767693.893843:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767693.893917:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767694.021873:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767694.021910:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767694.021914:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767694.021923:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4315 00000100:00080000:3.0:1775767694.021938:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767694.021943:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767694.021980:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967347840/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767694.022001:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767694.022004:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767694.022005:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767694.022008:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4315 00000100:00080000:3.0:1775767694.022011:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767694.022016:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767694.022024:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767694.022038:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967347840:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767694.022045:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967347968/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767694.022054:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767694.022057:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767694.022058:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767694.022063:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4315 00000100:00080000:3.0:1775767694.022068:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767694.022073:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767694.022080:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767694.022094:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967348096/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767694.022105:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1775767694.022113:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775767694.022121:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767694.022124:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767694.022128:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4315 00000100:00100000:0.0:1775767694.022145:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767694/real 1775767694] req@ffff8f9970edfb80 x1862022967347840/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767694.022158:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967347840/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767694.022166:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767694.022184:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767694.022189:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967347840:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775767694.022196:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767694.022202:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767694.022207:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767694.022210:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967347968:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767694.022231:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967348224/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767694.022234:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767694.022243:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767694/real 1775767694] req@ffff8f9970edc380 x1862022967347968/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767694.022251:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967347968/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767694.022257:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767694.022259:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767694.022262:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967347968:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767694.022279:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967348096:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767694.022297:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767694.022309:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767694/real 1775767694] req@ffff8f9970edca80 x1862022967348096/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767694.022324:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967348096/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767694.022333:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767694.022343:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767694.022351:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967348096:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767694.022370:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967348224:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767694.022389:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767694.022399:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767694/real 1775767694] req@ffff8f9970edea00 x1862022967348224/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767694.022411:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967348224/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767694.022418:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767694.022420:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767694.022422:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967348224:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767694.917848:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767694.917918:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767695.941844:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767695.941913:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767696.965826:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767696.965887:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767697.989855:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767697.989909:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767699.013813:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767699.013862:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767699.141867:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767699.141879:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767699.141884:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767699.141891:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4320 00000100:00080000:1.0:1775767699.141901:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767699.141906:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767699.141974:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967348352/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767699.142000:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767699.142002:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767699.142004:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767699.142007:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4320 00000100:00080000:1.0:1775767699.142010:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767699.142012:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767699.142020:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767699.142029:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967348480/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767699.142036:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767699.142039:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767699.142040:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767699.142042:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4320 00000100:00080000:1.0:1775767699.142044:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767699.142046:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767699.142049:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767699.142057:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967348352:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767699.142068:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967348608/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767699.142075:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767699.142077:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767699.142080:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767699.142086:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4320 00000100:00080000:1.0:1775767699.142088:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767699.142090:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767699.142093:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767699.142110:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967348736/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767699.142160:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767699.142201:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767699/real 1775767699] req@ffff8f994f2e6a00 x1862022967348352/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767699.142217:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967348352/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767699.142224:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767699.142227:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767699.142230:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967348352:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767699.142236:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967348480:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775767699.142252:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767699.142259:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767699/real 1775767699] req@ffff8f994f2e5f80 x1862022967348480/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767699.142267:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967348480/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767699.142273:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767699.142275:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767699.142277:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967348480:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767699.142302:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967348608:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767699.142322:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767699.142333:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767699/real 1775767699] req@ffff8f994f2e4000 x1862022967348608/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767699.142352:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967348608/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767699.142358:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767699.142360:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767699.142362:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967348608:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767699.142366:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967348736:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767699.142380:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767699.142386:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767699/real 1775767699] req@ffff8f994f2e6680 x1862022967348736/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767699.142393:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967348736/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767699.142398:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767699.142400:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767699.142402:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967348736:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767700.037897:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767700.037907:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767701.061846:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767701.061925:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767702.085855:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767702.085909:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767703.109866:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767703.109986:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767704.133851:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767704.133940:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767704.261825:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767704.261869:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767704.261873:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767704.261881:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4325 00000100:00080000:3.0:1775767704.261891:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767704.261896:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767704.261938:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddf80 x1862022967348864/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767704.261960:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767704.261963:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767704.261964:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767704.261975:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4325 00000100:00080000:3.0:1775767704.261978:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767704.261981:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767704.261989:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767704.261998:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd180 x1862022967348992/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767704.262012:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967348864:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775767704.262021:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767704.262024:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767704.262026:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767704.262030:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4325 00000100:00080000:3.0:1775767704.262033:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767704.262035:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767704.262039:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767704.262050:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967349120/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767704.262057:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767704.262059:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767704.262060:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767704.262062:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4325 00000100:00080000:3.0:1775767704.262065:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767704.262067:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767704.262084:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767704.262102:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967349248/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767704.262107:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767704.262148:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767704/real 1775767704] req@ffff8f9970eddf80 x1862022967348864/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767704.262195:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddf80 x1862022967348864/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767704.262206:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767704.262211:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767704.262216:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967348864:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767704.262240:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967348992:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775767704.262259:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767704.262268:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767704/real 1775767704] req@ffff8f9970edd180 x1862022967348992/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767704.262276:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd180 x1862022967348992/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767704.262282:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767704.262284:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767704.262286:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967348992:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767704.262292:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967349120:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767704.262311:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767704.262320:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767704/real 1775767704] req@ffff8f9970edea00 x1862022967349120/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767704.262333:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967349120/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767704.262366:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767704.262368:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767704.262371:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967349120:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767704.262375:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967349248:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767704.262391:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767704.262398:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767704/real 1775767704] req@ffff8f9970edca80 x1862022967349248/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767704.262405:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967349248/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767704.262411:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767704.262413:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767704.262415:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967349248:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767705.157827:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767705.157898:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767706.181839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767706.181891:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767707.205930:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767707.205937:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775767708.229835:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767708.229892:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767709.253804:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767709.253898:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767709.381843:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767709.381851:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767709.381855:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767709.381863:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4330 00000100:00080000:1.0:1775767709.381873:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767709.381912:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767709.381962:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967349376/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767709.381982:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767709.381984:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767709.381986:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767709.381988:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4330 00000100:00080000:1.0:1775767709.381991:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767709.381993:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767709.382000:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767709.382016:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967349504/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767709.382049:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767709.382052:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767709.382053:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767709.382056:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4330 00000100:00080000:1.0:1775767709.382061:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775767709.382062:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967349376:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775767709.382067:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767709.382075:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767709.382085:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967349632/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767709.382092:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767709.382094:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767709.382095:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767709.382097:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4330 00000100:00080000:1.0:1775767709.382099:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767709.382101:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767709.382104:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767709.382111:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967349760/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767709.382171:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967349504:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767709.382201:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967349632:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767709.382216:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967349760:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767710.277843:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767710.277904:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767711.301817:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767711.302078:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767712.325852:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767712.326042:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775767712.454338:0:2366:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:1.0:1775767712.454389:0:2366:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775767712.454404:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775767712.454459:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775767712.454471:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775767712.454496:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:0.0:1775767712.454506:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767709/real 1775767712] req@ffff8f994f2e4e00 x1862022967349632/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767712.454558:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967349632/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767712.454570:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767712.454576:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767712.454580:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967349632:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767712.454589:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767709/real 1775767712] req@ffff8f994f2e7b80 x1862022967349760/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767712.454600:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967349760/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767712.454607:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767712.454611:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767712.454614:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967349760:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767712.454621:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767709/real 1775767712] req@ffff8f994f2e4700 x1862022967349376/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767712.454634:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967349376/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767712.454642:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767712.454646:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767712.454649:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967349376:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767712.454656:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767709/real 1775767712] req@ffff8f994f2e6300 x1862022967349504/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767712.454686:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967349504/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767712.454696:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767712.454700:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767712.454703:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967349504:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767713.349805:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767713.349868:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767714.373876:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767714.373904:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00080000:1.0:1775767714.501872:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767714.501879:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767714.501883:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767714.501891:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4336 00000100:00080000:1.0:1775767714.501901:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767714.501906:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767714.501937:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967349888/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767714.501961:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767714.501965:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767714.501966:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767714.501987:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4336 00000100:00080000:1.0:1775767714.501991:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767714.501995:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767714.502005:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767714.502020:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967350016/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767714.502033:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775767714.502040:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967349888:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775767714.502040:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767714.502042:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767714.502054:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4336 00000100:00080000:1.0:1775767714.502058:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767714.502061:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767714.502067:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767714.502079:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967350144/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767714.502091:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767714.502094:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767714.502097:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767714.502101:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4336 00000100:00080000:1.0:1775767714.502107:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767714.502111:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:0.0:1775767714.502117:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:1.0:1775767714.502118:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767714.502134:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967350272/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767714.502170:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767714/real 1775767714] req@ffff8f994f2e6300 x1862022967349888/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767714.502191:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967349888/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767714.502201:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767714.502205:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767714.502209:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967349888:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767714.502235:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967350016:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775767714.502260:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767714.502279:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767714/real 1775767714] req@ffff8f994f2e4700 x1862022967350016/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767714.502309:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967350016/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767714.502318:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767714.502321:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767714.502325:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967350016:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767714.502332:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967350144:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767714.502354:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767714.502364:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767714/real 1775767714] req@ffff8f994f2e7b80 x1862022967350144/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767714.502376:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967350144/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767714.502384:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767714.502387:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767714.502390:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967350144:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767714.502396:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967350272:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767714.502415:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767714.502424:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767714/real 1775767714] req@ffff8f994f2e4e00 x1862022967350272/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767714.502434:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967350272/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767714.502440:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767714.502443:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767714.502483:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967350272:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767715.397843:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767715.397902:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767716.421808:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767716.421901:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767717.445855:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767717.445916:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767718.469834:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767718.469923:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767719.493903:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767719.493984:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767719.621870:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767719.621880:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767719.621885:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767719.621895:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4341 00000100:00080000:0.0:1775767719.621908:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767719.621915:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767719.621988:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967350400/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767719.622015:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767719.622018:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767719.622019:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767719.622023:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4341 00000100:00080000:0.0:1775767719.622027:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767719.622030:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767719.622044:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767719.622063:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967350400:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767719.622064:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967350528/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767719.622091:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767719.622095:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767719.622097:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767719.622103:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4341 00000100:00080000:0.0:1775767719.622107:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767719.622110:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767719.622128:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767719.622157:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967350656/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767719.622167:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767719.622170:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767719.622171:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767719.622175:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4341 00000100:00080000:0.0:1775767719.622179:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767719.622182:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767719.622186:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767719.622197:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967350784/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767719.622351:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767719.622399:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767719/real 1775767719] req@ffff8f9970a76a00 x1862022967350400/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767719.622412:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967350400/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767719.622420:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767719.622422:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767719.622426:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967350400:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767719.622448:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967350528:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775767719.622467:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767719.622475:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767719/real 1775767719] req@ffff8f9970a77100 x1862022967350528/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767719.622483:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967350528/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767719.622510:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767719.622513:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767719.622515:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967350528:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767719.622519:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967350656:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767719.622533:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767719.622539:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767719/real 1775767719] req@ffff8f9970a75c00 x1862022967350656/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767719.622547:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967350656/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767719.622552:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767719.622554:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767719.622556:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967350656:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767719.622559:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967350784:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767719.622569:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767719.622574:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767719/real 1775767719] req@ffff8f9970a77480 x1862022967350784/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767719.622581:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967350784/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767719.622586:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767719.622587:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767719.622590:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967350784:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767720.517831:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767720.517898:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767721.541929:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767721.541938:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767722.565856:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767722.565933:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767723.589859:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767723.589926:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767724.613847:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767724.613967:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767724.741882:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767724.741892:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767724.741897:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767724.741906:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4346 00000100:00080000:3.0:1775767724.741941:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767724.741948:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767724.741997:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967350912/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767724.742023:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767724.742026:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767724.742027:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767724.742032:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4346 00000100:00080000:3.0:1775767724.742035:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767724.742038:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767724.742048:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767724.742072:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967350912:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767724.742073:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967351040/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767724.742091:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767724.742095:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767724.742097:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767724.742101:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4346 00000100:00080000:3.0:1775767724.742105:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767724.742109:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767724.742114:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775767724.742135:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767724.742141:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967351168/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767724.742152:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767724.742165:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767724.742166:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767724.742170:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4346 00000100:00080000:3.0:1775767724.742175:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775767724.742178:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767724/real 1775767724] req@ffff8f9970edc380 x1862022967350912/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767724.742182:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767724.742191:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767724.742202:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967350912/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767724.742211:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767724.742216:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767724.742217:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967351296/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767724.742221:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967350912:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767724.742245:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967351040:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775767724.742261:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767724.742268:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767724/real 1775767724] req@ffff8f9970edfb80 x1862022967351040/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767724.742276:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967351040/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767724.742281:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767724.742284:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767724.742286:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967351040:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767724.742289:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967351168:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767724.742319:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767724.742326:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767724/real 1775767724] req@ffff8f9970ede300 x1862022967351168/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767724.742334:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967351168/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767724.742339:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767724.742341:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767724.742344:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967351168:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767724.742348:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967351296:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767724.742357:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767724.742363:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767724/real 1775767724] req@ffff8f9970edd880 x1862022967351296/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767724.742370:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967351296/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767724.742375:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767724.742376:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767724.742378:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967351296:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767725.637841:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767725.637910:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767726.661810:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767726.661896:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767727.685866:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767727.685924:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767728.709889:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767728.709939:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767729.733871:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767729.733938:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767729.861847:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767729.861855:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767729.861860:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767729.861870:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4351 00000100:00080000:3.0:1775767729.861882:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767729.861890:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767729.861927:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967351424/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767729.861955:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767729.861958:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767729.861959:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767729.861963:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4351 00000100:00080000:3.0:1775767729.861967:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767729.861972:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767729.861982:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767729.861997:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967351552/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767729.862018:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967351424:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775767729.862036:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767729.862039:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767729.862041:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767729.862045:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4351 00000100:00080000:3.0:1775767729.862049:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767729.862053:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767729.862058:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767729.862071:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967351680/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767729.862078:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767729.862081:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767729.862083:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767729.862085:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4351 00000100:00080000:3.0:1775767729.862088:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767729.862091:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767729.862095:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775767729.862096:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767729.862104:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967351808/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767729.862135:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767729/real 1775767729] req@ffff8f9970edf480 x1862022967351424/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767729.862174:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967351424/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767729.862182:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767729.862184:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767729.862188:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967351424:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767729.862207:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967351552:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775767729.862227:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767729.862237:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767729/real 1775767729] req@ffff8f9970edf800 x1862022967351552/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767729.862245:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967351552/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767729.862251:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767729.862253:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767729.862255:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967351552:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767729.862259:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967351680:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767729.862281:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767729.862287:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767729/real 1775767729] req@ffff8f9970eded80 x1862022967351680/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767729.862295:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967351680/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767729.862305:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767729.862308:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767729.862310:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967351680:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767729.862314:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967351808:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767729.862326:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767729.862333:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767729/real 1775767729] req@ffff8f9970ede680 x1862022967351808/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767729.862342:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967351808/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767729.862349:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767729.862352:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767729.862354:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967351808:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767730.757845:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767730.757901:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767731.781863:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767731.781946:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767732.805847:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767732.805916:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767733.829841:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767733.829915:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767734.853848:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767734.853922:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767734.982201:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767734.982213:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767734.982218:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767734.982230:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4356 00000100:00080000:3.0:1775767734.982292:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767734.982301:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767734.982343:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967351936/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767734.982391:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767734.982396:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767734.982399:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767734.982405:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4356 00000100:00080000:3.0:1775767734.982410:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767734.982414:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767734.982425:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767734.982439:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967352064/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767734.982450:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767734.982453:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767734.982454:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767734.982458:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4356 00000100:00080000:3.0:1775767734.982462:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767734.982466:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767734.982478:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767734.982525:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf100 x1862022967352192/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767734.982537:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767734.982558:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767734.982560:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767734.982565:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4356 00000100:00080000:3.0:1775767734.982569:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767734.982572:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767734.982578:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767734.982591:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967352320/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767734.982674:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967351936:192.168.202.146@tcp:250:kworker.0 00000800:00000100:2.0:1775767734.982864:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767734.982948:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767734/real 1775767734] req@ffff8f9970edc700 x1862022967351936/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767734.982971:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967351936/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767734.982984:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767734.982989:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767734.982994:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967351936:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767734.983014:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967352064:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775767734.983038:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767734.983050:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767734/real 1775767734] req@ffff8f9970edc000 x1862022967352064/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767734.983062:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967352064/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767734.983072:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767734.983075:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767734.983079:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967352064:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767734.983085:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967352192:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767734.983103:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767734.983113:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767734/real 1775767734] req@ffff8f9970edf100 x1862022967352192/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767734.983126:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf100 x1862022967352192/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767734.983134:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767734.983137:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767734.983140:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967352192:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767734.983145:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967352320:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767734.983163:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767734.983172:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767734/real 1775767734] req@ffff8f9970edd500 x1862022967352320/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767734.983183:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967352320/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767734.983191:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767734.983194:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767734.983198:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967352320:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767735.877864:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767735.877893:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767736.901862:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767736.901942:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767737.925848:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767737.925890:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767738.949844:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767738.949936:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767739.973811:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767739.973870:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767740.101858:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767740.101866:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767740.101870:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767740.101879:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4361 00000100:00080000:1.0:1775767740.101895:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767740.101901:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767740.101934:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967352448/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767740.101956:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767740.101959:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767740.101960:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767740.101963:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4361 00000100:00080000:1.0:1775767740.101966:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767740.101968:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767740.101975:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767740.101984:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967352576/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767740.101992:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767740.101997:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1775767740.101997:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967352448:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775767740.101999:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767740.102007:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4361 00000100:00080000:1.0:1775767740.102011:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767740.102015:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767740.102020:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775767740.102056:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767740.102082:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967352704/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767740.102090:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767740.102092:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767740.102094:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767740.102099:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4361 00000100:00100000:2.0:1775767740.102099:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767740/real 1775767740] req@ffff8f994f2e6680 x1862022967352448/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767740.102104:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767740.102112:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767740.102118:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767740.102122:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967352448/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767740.102129:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967352832/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767740.102134:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767740.102141:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767740.102144:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967352448:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767740.102166:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967352576:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775767740.102181:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767740.102190:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767740/real 1775767740] req@ffff8f994f2e4000 x1862022967352576/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767740.102207:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967352576/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767740.102216:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767740.102219:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767740.102221:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967352576:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767740.102225:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967352704:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767740.102240:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767740.102246:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767740/real 1775767740] req@ffff8f994f2e5f80 x1862022967352704/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767740.102254:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967352704/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767740.102259:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767740.102261:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767740.102263:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967352704:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767740.102267:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967352832:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767740.102276:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767740.102281:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767740/real 1775767740] req@ffff8f994f2e6a00 x1862022967352832/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767740.102289:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967352832/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767740.102293:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767740.102295:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767740.102297:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967352832:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767740.997867:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767740.997932:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767742.021830:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767742.021885:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767743.045889:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767743.045907:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767744.069820:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767744.069875:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767745.093851:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767745.093907:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767745.221844:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767745.221854:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767745.221859:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767745.221869:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4366 00000100:00080000:3.0:1775767745.221881:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767745.221889:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767745.221926:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967352960/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767745.221947:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767745.221950:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767745.221952:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767745.221955:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4366 00000100:00080000:3.0:1775767745.221958:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767745.221961:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767745.221969:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767745.221979:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967352960:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767745.221980:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddc00 x1862022967353088/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767745.222010:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767745.222013:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767745.222014:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767745.222017:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4366 00000100:00080000:3.0:1775767745.222026:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767745.222029:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767745.222034:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775767745.222038:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767745.222048:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967353216/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767745.222055:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767745.222058:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767745.222059:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767745.222062:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4366 00000100:00080000:3.0:1775767745.222064:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767745.222066:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767745.222069:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767745.222075:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf100 x1862022967353344/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767745.222101:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767745/real 1775767745] req@ffff8f9970edce00 x1862022967352960/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767745.222125:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967352960/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767745.222136:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767745.222139:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767745.222142:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967352960:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767745.222172:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967353088:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775767745.222189:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767745.222198:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767745/real 1775767745] req@ffff8f9970eddc00 x1862022967353088/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767745.222206:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddc00 x1862022967353088/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767745.222211:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767745.222213:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767745.222215:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967353088:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767745.222219:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967353216:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767745.222230:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767745.222235:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767745/real 1775767745] req@ffff8f9970edd500 x1862022967353216/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767745.222247:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967353216/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767745.222252:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767745.222253:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767745.222255:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967353216:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767745.222258:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967353344:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767745.222268:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767745.222273:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767745/real 1775767745] req@ffff8f9970edf100 x1862022967353344/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767745.222280:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf100 x1862022967353344/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767745.222285:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767745.222287:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767745.222288:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967353344:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767746.117840:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767746.117904:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767747.141840:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767747.141912:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767748.165808:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767748.165894:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767749.189835:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767749.189891:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767750.213876:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767750.213886:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00080000:0.0:1775767750.341832:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767750.341839:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767750.341843:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767750.341853:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4371 00000100:00080000:0.0:1775767750.341863:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767750.341869:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767750.341901:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967353472/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767750.341923:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767750.341926:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767750.341928:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767750.341931:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4371 00000100:00080000:0.0:1775767750.341934:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767750.341937:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767750.341945:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767750.341955:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967353472:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767750.341956:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967353600/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767750.341969:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767750.341972:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767750.341973:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767750.341975:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4371 00000100:00080000:0.0:1775767750.341978:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767750.341980:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767750.341984:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767750.342001:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967353728/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767750.342009:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767750.342012:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767750.342013:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767750.342015:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4371 00000100:00080000:0.0:1775767750.342018:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775767750.342020:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775767750.342024:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767750.342028:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767750.342039:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967353856/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767750.342080:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767750/real 1775767750] req@ffff8f9970a75180 x1862022967353472/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767750.342103:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967353472/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767750.342115:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767750.342120:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767750.342125:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967353472:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767750.342152:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967353600:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775767750.342171:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767750.342180:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767750/real 1775767750] req@ffff8f9970a76300 x1862022967353600/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767750.342189:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967353600/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767750.342194:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767750.342196:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767750.342198:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967353600:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767750.342202:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967353728:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767750.342214:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767750.342219:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767750/real 1775767750] req@ffff8f9970a75500 x1862022967353728/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767750.342227:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967353728/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767750.342236:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767750.342238:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767750.342240:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967353728:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767750.342244:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967353856:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767750.342253:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767750.342258:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767750/real 1775767750] req@ffff8f9970a77480 x1862022967353856/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767750.342265:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967353856/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767750.342270:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767750.342272:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767750.342274:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967353856:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767751.237845:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767751.237903:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767752.261853:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767752.261930:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767753.285852:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767753.285910:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767754.309823:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767754.309855:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767755.333826:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767755.333881:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767755.461846:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767755.461857:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767755.461862:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767755.461872:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4377 00000100:00080000:0.0:1775767755.461884:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767755.461891:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767755.461930:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967353984/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767755.461962:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767755.461967:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767755.461970:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767755.461976:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4377 00000100:00080000:0.0:1775767755.461981:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767755.461986:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767755.461992:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967353984:192.168.202.146@tcp:250:kworker.0 00000008:00100000:0.0:1775767755.461997:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767755.462014:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967354112/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767755.462028:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767755.462032:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767755.462035:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767755.462039:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4377 00000100:00080000:0.0:1775767755.462054:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775767755.462060:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775767755.462061:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767755.462069:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767755.462084:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967354240/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767755.462095:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767755.462099:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767755.462101:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767755.462106:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4377 00000100:00080000:0.0:1775767755.462110:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767755.462114:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767755.462117:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767755/real 1775767755] req@ffff8f9970a75c00 x1862022967353984/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1775767755.462122:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767755.462142:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967353984/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767755.462152:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767755.462155:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767755.462158:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967353984:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767755.462177:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967354112:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767755.462182:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967354368/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767755.462199:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767755.462209:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767755/real 1775767755] req@ffff8f9970a77100 x1862022967354112/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767755.462218:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967354112/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767755.462224:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767755.462226:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767755.462228:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967354112:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767755.462232:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967354240:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767755.462244:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767755.462249:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767755/real 1775767755] req@ffff8f9970a76a00 x1862022967354240/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767755.462267:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967354240/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767755.462275:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767755.462279:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767755.462282:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967354240:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767755.462298:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967354368:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767755.462319:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767755.462328:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767755/real 1775767755] req@ffff8f9970a74380 x1862022967354368/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767755.462336:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967354368/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767755.462342:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767755.462344:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767755.462346:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967354368:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767756.357841:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767756.357906:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767757.381881:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767757.381919:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767758.405876:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767758.405957:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767759.429877:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767759.429939:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767760.453853:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767760.453915:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767760.581857:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767760.581865:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767760.581869:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767760.581877:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4382 00000100:00080000:1.0:1775767760.581887:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767760.581892:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767760.581927:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967354496/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767760.581950:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767760.581953:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767760.581954:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767760.581957:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4382 00000100:00080000:1.0:1775767760.581959:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767760.581962:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767760.581972:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767760.581983:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967354496:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767760.581988:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967354624/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767760.582002:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767760.582006:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767760.582008:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767760.582012:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4382 00000100:00080000:1.0:1775767760.582017:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767760.582022:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767760.582027:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767760.582038:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967354752/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767760.582045:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767760.582048:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767760.582051:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1775767760.582051:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775767760.582057:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4382 00000100:00080000:1.0:1775767760.582062:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767760.582065:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767760.582069:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767760.582077:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967354880/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767760.582098:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767760/real 1775767760] req@ffff8f994f2e6d80 x1862022967354496/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767760.582115:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967354496/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767760.582123:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767760.582126:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767760.582129:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967354496:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767760.582155:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967354624:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775767760.582170:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767760.582178:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767760/real 1775767760] req@ffff8f994f2e4a80 x1862022967354624/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767760.582186:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967354624/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767760.582192:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767760.582194:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767760.582196:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967354624:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767760.582200:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967354752:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767760.582218:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767760.582228:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767760/real 1775767760] req@ffff8f994f2e7800 x1862022967354752/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767760.582240:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967354752/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767760.582249:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767760.582253:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767760.582256:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967354752:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767760.582262:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967354880:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767760.582324:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767760.582337:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767760/real 1775767760] req@ffff8f994f2e5180 x1862022967354880/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767760.582350:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967354880/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767760.582359:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767760.582362:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767760.582365:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967354880:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767761.477844:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767761.477906:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767762.501837:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767762.501890:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767763.525840:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767763.525900:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767764.549875:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767764.549884:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767765.573860:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767765.573931:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767765.701838:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767765.701850:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767765.701855:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767765.701866:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4387 00000100:00080000:1.0:1775767765.701878:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767765.701886:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767765.701924:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967355008/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767765.701958:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767765.701961:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767765.701963:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767765.701967:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4387 00000100:00080000:1.0:1775767765.701972:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767765.701976:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767765.701984:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767765.702001:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967355008:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767765.702002:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967355136/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767765.702064:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767765.702068:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767765.702070:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767765.702075:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4387 00000100:00080000:1.0:1775767765.702080:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767765.702085:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767765.702092:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775767765.702101:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767765.702108:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967355264/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767765.702120:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767765.702124:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767765.702127:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767765.702131:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4387 00000100:00080000:1.0:1775767765.702136:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767765.702139:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767765.702143:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767765.702153:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967355392/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767765.702226:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767765/real 1775767765] req@ffff8f994f2e5180 x1862022967355008/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767765.702276:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967355008/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767765.702407:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767765.702413:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767765.702417:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967355008:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767765.702504:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967355136:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767765.702526:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767765.702536:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767765/real 1775767765] req@ffff8f994f2e7800 x1862022967355136/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767765.702545:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967355136/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767765.702551:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767765.702553:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767765.702555:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967355136:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767765.702567:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967355264:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767765.702580:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767765.702587:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767765/real 1775767765] req@ffff8f994f2e4a80 x1862022967355264/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767765.702595:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967355264/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767765.702617:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767765.702619:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767765.702622:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967355264:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767765.702626:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967355392:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767765.702639:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767765.702645:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767765/real 1775767765] req@ffff8f994f2e6d80 x1862022967355392/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767765.702653:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967355392/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767765.702658:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767765.702660:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767765.702662:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967355392:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767766.597847:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767766.597902:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767767.621837:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767767.621879:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767768.645836:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767768.645927:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767769.669790:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767769.669866:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767770.693817:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767770.693878:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767770.821834:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767770.821843:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767770.821848:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767770.821858:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4392 00000100:00080000:2.0:1775767770.821870:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767770.821877:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767770.821917:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2680 x1862022967355520/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767770.821942:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767770.821946:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767770.821949:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767770.821954:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4392 00000100:00080000:2.0:1775767770.821958:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767770.821964:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767770.821973:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767770.821988:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967355520:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767770.821989:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0000 x1862022967355648/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767770.822004:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767770.822007:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767770.822008:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767770.822011:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4392 00000100:00080000:2.0:1775767770.822014:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767770.822017:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767770.822022:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767770.822031:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0380 x1862022967355776/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767770.822038:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767770.822040:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767770.822042:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767770.822044:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4392 00000100:00080000:2.0:1775767770.822047:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767770.822049:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767770.822052:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775767770.822054:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767770.822062:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3800 x1862022967355904/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767770.822119:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767770/real 1775767770] req@ffff8f99703d2680 x1862022967355520/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767770.822140:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2680 x1862022967355520/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767770.822151:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767770.822155:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767770.822159:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967355520:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767770.822217:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967355648:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767770.822240:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767770.822254:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767770/real 1775767770] req@ffff8f99703d0000 x1862022967355648/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767770.822268:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0000 x1862022967355648/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767770.822275:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767770.822285:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767770.822288:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967355648:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767770.822292:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967355776:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767770.822309:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767770.822316:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767770/real 1775767770] req@ffff8f99703d0380 x1862022967355776/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767770.822324:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0380 x1862022967355776/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767770.822329:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767770.822331:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767770.822333:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967355776:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767770.822337:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967355904:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767770.822354:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767770.822360:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767770/real 1775767770] req@ffff8f99703d3800 x1862022967355904/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767770.822383:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3800 x1862022967355904/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767770.822389:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767770.822391:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767770.822393:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967355904:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767771.717884:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767771.717896:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767772.741846:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767772.741904:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767773.765843:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767773.765938:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767774.789829:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767774.789884:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767775.813835:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767775.813908:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767775.941826:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767775.941837:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767775.941842:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767775.941853:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4397 00000100:00080000:2.0:1775767775.941865:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767775.941872:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767775.941910:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1500 x1862022967356032/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767775.941935:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767775.941938:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767775.941939:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767775.941942:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4397 00000100:00080000:2.0:1775767775.941945:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767775.941948:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767775.941956:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767775.941965:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2d80 x1862022967356160/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767775.941972:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767775.941975:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767775.941977:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767775.941979:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4397 00000100:00080000:2.0:1775767775.941982:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775767775.941985:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967356032:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775767775.941988:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767775.941996:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767775.942010:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0e00 x1862022967356288/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767775.942021:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767775.942025:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767775.942027:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767775.942031:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4397 00000100:00080000:2.0:1775767775.942036:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767775.942040:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767775.942046:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767775.942059:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0700 x1862022967356416/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767775.942084:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967356160:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767775.942140:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967356288:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767775.942167:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967356416:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767776.837810:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767776.837903:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767777.861846:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767777.861907:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767778.885868:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767778.885869:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775767779.014094:0:2368:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:2.0:1775767779.014142:0:2368:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775767779.014160:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:2.0:1775767779.014201:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:2.0:1775767779.014211:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:2.0:1775767779.014220:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:3.0:1775767779.014422:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767775/real 1775767779] req@ffff8f99703d0e00 x1862022967356288/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767779.014451:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0e00 x1862022967356288/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767779.014462:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767779.014468:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767779.014472:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967356288:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767779.014480:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767775/real 1775767779] req@ffff8f99703d0700 x1862022967356416/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767779.014492:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0700 x1862022967356416/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767779.014568:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767779.014572:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767779.014576:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967356416:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767779.014585:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767775/real 1775767779] req@ffff8f99703d1500 x1862022967356032/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767779.014599:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1500 x1862022967356032/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767779.014606:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767779.014609:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767779.014612:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967356032:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767779.014619:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767775/real 1775767779] req@ffff8f99703d2d80 x1862022967356160/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767779.014630:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2d80 x1862022967356160/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767779.014638:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767779.014642:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767779.014645:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967356160:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767779.909853:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767779.909912:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767780.933839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767780.933892:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767781.061901:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767781.061909:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767781.061913:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767781.061922:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4402 00000100:00080000:1.0:1775767781.061933:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767781.061939:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767781.061973:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967356544/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767781.062020:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767781.062023:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767781.062026:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767781.062030:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4402 00000100:00080000:1.0:1775767781.062035:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767781.062039:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767781.062049:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767781.062052:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967356544:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767781.062061:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967356672/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767781.062072:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767781.062075:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767781.062077:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767781.062080:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4402 00000100:00080000:1.0:1775767781.062083:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767781.062087:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767781.062092:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767781.062102:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967356800/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767781.062111:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1775767781.062114:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775767781.062115:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767781.062117:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767781.062120:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4402 00000100:00080000:1.0:1775767781.062123:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767781.062126:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767781.062130:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767781.062138:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967356928/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767781.062166:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767781/real 1775767781] req@ffff8f994f2e6a00 x1862022967356544/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767781.062230:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967356544/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767781.062241:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767781.062246:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767781.062249:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967356544:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767781.062275:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967356672:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767781.062300:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767781.062315:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767781/real 1775767781] req@ffff8f994f2e5f80 x1862022967356672/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767781.062365:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967356672/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767781.062374:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767781.062377:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767781.062380:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967356672:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767781.062386:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967356800:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767781.062407:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767781.062419:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767781/real 1775767781] req@ffff8f994f2e4000 x1862022967356800/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767781.062431:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967356800/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767781.062462:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767781.062466:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767781.062470:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967356800:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767781.062477:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967356928:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767781.062499:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767781.062511:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767781/real 1775767781] req@ffff8f994f2e6680 x1862022967356928/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767781.062521:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967356928/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767781.062529:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767781.062626:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767781.062632:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967356928:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767781.957882:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767781.957961:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767782.981870:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767782.981963:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767784.005839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767784.005906:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767785.029874:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767785.029930:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767786.053932:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767786.053977:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767786.181891:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767786.181899:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767786.181903:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767786.181911:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4407 00000100:00080000:2.0:1775767786.181921:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767786.181926:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767786.181962:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3100 x1862022967357056/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767786.181994:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767786.182006:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767786.182008:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767786.182019:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4407 00000100:00080000:2.0:1775767786.182023:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767786.182027:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767786.182035:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767786.182048:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2300 x1862022967357184/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767786.182065:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967357056:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775767786.182072:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767786.182077:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767786.182078:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767786.182082:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4407 00000100:00080000:2.0:1775767786.182084:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767786.182087:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767786.182090:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767786.182103:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1180 x1862022967357312/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767786.182110:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767786.182113:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767786.182114:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767786.182116:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4407 00000100:00080000:2.0:1775767786.182119:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767786.182121:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767786.182124:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767786.182131:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3480 x1862022967357440/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775767786.182171:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767786.182240:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767786/real 1775767786] req@ffff8f99703d3100 x1862022967357056/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767786.182266:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3100 x1862022967357056/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767786.182293:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767786.182299:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767786.182304:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967357056:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767786.182501:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967357184:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767786.182552:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767786.182580:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767786/real 1775767786] req@ffff8f99703d2300 x1862022967357184/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767786.182593:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2300 x1862022967357184/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767786.182601:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767786.182604:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767786.182606:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967357184:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767786.182611:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967357312:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767786.182625:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767786.182631:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767786/real 1775767786] req@ffff8f99703d1180 x1862022967357312/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767786.182638:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1180 x1862022967357312/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767786.182644:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767786.182645:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767786.182647:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967357312:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767786.182651:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967357440:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767786.182662:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767786.182667:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767786/real 1775767786] req@ffff8f99703d3480 x1862022967357440/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767786.182697:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3480 x1862022967357440/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767786.182702:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767786.182704:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767786.182707:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967357440:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767787.077850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767787.077890:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767788.101837:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767788.101899:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767789.125823:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767789.125866:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767790.149857:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767790.149901:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767791.173857:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767791.173929:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767791.301859:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767791.301868:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767791.301872:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767791.301882:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4412 00000100:00080000:2.0:1775767791.301892:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767791.301897:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767791.301932:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3b80 x1862022967357568/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767791.301956:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767791.301959:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767791.301960:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767791.301964:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4412 00000100:00080000:2.0:1775767791.301966:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767791.301969:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767791.301978:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767791.301987:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1880 x1862022967357696/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767791.301991:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967357568:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775767791.302001:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767791.302004:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767791.302005:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767791.302007:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4412 00000100:00080000:2.0:1775767791.302009:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767791.302011:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767791.302015:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767791.302024:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2a00 x1862022967357824/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767791.302031:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767791.302033:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767791.302035:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767791.302037:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4412 00000100:00080000:2.0:1775767791.302047:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767791.302049:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767791.302052:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775767791.302059:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767791.302061:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1c00 x1862022967357952/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767791.302109:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767791/real 1775767791] req@ffff8f99703d3b80 x1862022967357568/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767791.302125:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3b80 x1862022967357568/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767791.302134:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767791.302139:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767791.302142:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967357568:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767791.302165:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967357696:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767791.302188:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767791.302215:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767791/real 1775767791] req@ffff8f99703d1880 x1862022967357696/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767791.302245:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1880 x1862022967357696/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767791.302300:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767791.302302:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767791.302305:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967357696:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767791.302310:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967357824:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767791.302327:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767791.302334:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767791/real 1775767791] req@ffff8f99703d2a00 x1862022967357824/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767791.302342:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2a00 x1862022967357824/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767791.302347:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767791.302349:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767791.302351:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967357824:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767791.302354:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967357952:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767791.302364:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767791.302369:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767791/real 1775767791] req@ffff8f99703d1c00 x1862022967357952/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767791.302376:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1c00 x1862022967357952/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767791.302381:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767791.302382:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767791.302394:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967357952:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767792.197835:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767792.197973:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767793.221905:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767793.221908:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767794.245826:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767794.245891:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767795.269835:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767795.269925:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767796.293816:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767796.293917:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767796.421885:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767796.421896:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767796.421901:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767796.421912:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4418 00000100:00080000:3.0:1775767796.421924:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767796.421931:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767796.421981:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967358080/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767796.422010:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767796.422013:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767796.422014:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767796.422018:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4418 00000100:00080000:3.0:1775767796.422022:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767796.422026:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767796.422039:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767796.422051:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967358208/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767796.422061:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775767796.422065:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967358080:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775767796.422065:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767796.422067:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767796.422082:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4418 00000100:00080000:3.0:1775767796.422086:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775767796.422151:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767796.422231:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767796/real 1775767796] req@ffff8f9970edc000 x1862022967358080/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767796.422234:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767796.422248:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767796.422255:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967358080/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767796.422264:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767796.422269:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767796.422273:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967358336/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767796.422276:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967358080:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775767796.422293:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767796.422296:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767796.422298:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775767796.422302:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967358208:192.168.202.146@tcp:38:kworker.0 00000100:00080000:3.0:1775767796.422303:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4418 00000100:00080000:3.0:1775767796.422311:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767796.422314:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767796.422318:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775767796.422331:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767796.422332:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967358464/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767796.422344:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767796/real 1775767796] req@ffff8f9970edc700 x1862022967358208/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767796.422354:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967358208/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767796.422359:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767796.422362:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767796.422365:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967358208:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767796.422370:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967358336:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767796.422383:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767796.422388:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767796/real 1775767796] req@ffff8f9970ede680 x1862022967358336/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767796.422395:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967358336/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767796.422400:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767796.422402:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767796.422404:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967358336:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767796.422412:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967358464:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767796.422422:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767796.422428:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767796/real 1775767796] req@ffff8f9970eded80 x1862022967358464/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767796.422435:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967358464/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767796.422440:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767796.422442:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767796.422444:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967358464:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767797.317800:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767797.317851:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767798.341811:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767798.341856:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767799.365827:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767799.365860:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767800.389892:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767800.389899:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767801.413886:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767801.413978:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767801.541966:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767801.541980:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767801.541987:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767801.542005:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4423 00000100:00080000:2.0:1775767801.542041:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767801.542052:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767801.542118:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1c00 x1862022967358592/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767801.542183:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767801.542188:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767801.542190:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767801.542196:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4423 00000100:00080000:2.0:1775767801.542201:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767801.542212:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767801.542225:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767801.542241:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2a00 x1862022967358720/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767801.542264:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767801.542269:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767801.542271:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767801.542276:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4423 00000100:00080000:2.0:1775767801.542284:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767801.542290:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767801.542297:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767801.542313:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1880 x1862022967358848/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767801.542325:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767801.542329:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767801.542332:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767801.542337:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4423 00000100:00080000:2.0:1775767801.542342:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767801.542348:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767801.542353:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767801.542364:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3b80 x1862022967358976/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767801.542687:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967358592:192.168.202.146@tcp:250:kworker.0 00000800:00000100:3.0:1775767801.542867:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767801.542908:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767801/real 1775767801] req@ffff8f99703d1c00 x1862022967358592/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767801.542923:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1c00 x1862022967358592/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767801.542932:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767801.542935:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767801.542938:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967358592:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767801.543049:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967358720:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767801.543077:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767801.543091:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767801/real 1775767801] req@ffff8f99703d2a00 x1862022967358720/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767801.543100:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2a00 x1862022967358720/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767801.543106:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767801.543108:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767801.543110:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967358720:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767801.543114:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967358848:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767801.543163:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767801.543175:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767801/real 1775767801] req@ffff8f99703d1880 x1862022967358848/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767801.543188:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1880 x1862022967358848/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767801.543195:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767801.543196:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767801.543198:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967358848:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767801.543202:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967358976:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767801.543218:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767801.543225:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767801/real 1775767801] req@ffff8f99703d3b80 x1862022967358976/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767801.543265:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3b80 x1862022967358976/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767801.543269:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767801.543271:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767801.543273:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967358976:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767802.437867:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767802.437999:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767803.461890:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767803.461982:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767804.485835:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767804.485886:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767805.509874:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767805.509930:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767806.533887:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767806.533902:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767806.661905:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767806.661913:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767806.661917:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767806.661927:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4428 00000100:00080000:0.0:1775767806.661937:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767806.661943:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767806.661976:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967359104/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767806.662005:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767806.662008:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767806.662009:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767806.662013:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4428 00000100:00080000:0.0:1775767806.662015:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767806.662018:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767806.662026:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767806.662042:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967359232/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767806.662049:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767806.662052:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767806.662053:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767806.662055:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4428 00000100:00080000:0.0:1775767806.662057:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767806.662059:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767806.662064:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767806.662070:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967359360/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767806.662077:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767806.662095:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767806.662096:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767806.662099:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4428 00000100:00080000:0.0:1775767806.662102:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767806.662104:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767806.662108:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767806.662112:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967359104:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767806.662118:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967359488/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775767806.662172:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767806.662222:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767806/real 1775767806] req@ffff8f9970a74700 x1862022967359104/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767806.662237:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967359104/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767806.662245:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767806.662249:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767806.662253:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967359104:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767806.662259:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967359232:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767806.662274:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767806.662282:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767806/real 1775767806] req@ffff8f9970a76d80 x1862022967359232/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767806.662290:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967359232/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767806.662418:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767806.662425:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767806.662429:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967359232:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767806.662440:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967359360:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767806.662495:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767806.662520:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767806/real 1775767806] req@ffff8f9970a77800 x1862022967359360/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767806.662539:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967359360/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767806.662549:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767806.662553:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767806.662557:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967359360:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767806.662591:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967359488:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767806.662609:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767806.662618:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767806/real 1775767806] req@ffff8f9970a74380 x1862022967359488/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767806.662627:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967359488/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767806.662633:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767806.662636:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767806.662727:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967359488:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767807.557914:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767807.557958:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767808.581888:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767808.581889:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775767809.605836:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767809.605937:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767810.629815:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767810.629878:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767811.653858:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767811.653940:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767811.781841:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767811.781855:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767811.781861:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767811.781874:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4433 00000100:00080000:3.0:1775767811.781889:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767811.781897:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767811.781949:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967359616/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767811.781984:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767811.781988:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767811.781990:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767811.781995:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4433 00000100:00080000:3.0:1775767811.781999:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767811.782003:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767811.782014:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767811.782044:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967359744/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767811.782057:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767811.782061:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767811.782063:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775767811.782069:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967359616:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775767811.782070:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4433 00000100:00080000:3.0:1775767811.782076:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767811.782082:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767811.782088:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767811.782103:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967359872/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767811.782113:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767811.782117:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767811.782119:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767811.782123:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4433 00000100:00080000:3.0:1775767811.782142:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767811.782146:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767811.782152:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775767811.782161:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767811.782166:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967360000/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767811.782228:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767811/real 1775767811] req@ffff8f9970edf800 x1862022967359616/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767811.782254:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967359616/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767811.782266:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767811.782272:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767811.782276:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967359616:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767811.782304:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967359744:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775767811.782327:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767811.782338:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767811/real 1775767811] req@ffff8f9970edf480 x1862022967359744/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767811.782352:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967359744/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767811.782360:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767811.782364:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767811.782368:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967359744:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767811.782374:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967359872:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767811.782391:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767811.782400:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767811/real 1775767811] req@ffff8f9970edd880 x1862022967359872/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767811.782412:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967359872/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767811.782421:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767811.782424:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767811.782428:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967359872:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767811.782434:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967360000:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767811.782450:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767811.782459:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767811/real 1775767811] req@ffff8f9970ede300 x1862022967360000/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767811.782471:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967360000/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767811.782480:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767811.782484:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767811.782487:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967360000:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767812.677875:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767812.677986:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767813.701832:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767813.701889:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767814.725922:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767814.725951:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767815.749832:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767815.749899:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767816.773871:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767816.773948:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767816.901874:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767816.901883:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767816.901887:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767816.901896:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4438 00000100:00080000:1.0:1775767816.901906:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767816.901912:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767816.901946:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967360128/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767816.901973:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767816.901976:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767816.901977:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767816.901992:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4438 00000100:00080000:1.0:1775767816.901996:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767816.902000:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767816.902008:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767816.902018:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967360256/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767816.902027:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767816.902029:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767816.902031:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767816.902042:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4438 00000100:00100000:2.0:1775767816.902042:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967360128:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775767816.902047:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767816.902056:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767816.902059:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767816.902065:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967360384/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767816.902100:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767816.902102:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767816.902104:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767816.902107:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4438 00000100:00080000:1.0:1775767816.902111:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767816.902113:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767816.902117:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767816.902124:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967360512/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767816.902146:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767816.902206:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767816/real 1775767816] req@ffff8f994f2e4e00 x1862022967360128/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767816.902228:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967360128/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767816.902238:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767816.902241:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767816.902245:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967360128:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767816.902272:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967360256:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775767816.902291:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767816.902300:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767816/real 1775767816] req@ffff8f994f2e7b80 x1862022967360256/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767816.902308:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967360256/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767816.902314:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767816.902316:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767816.902319:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967360256:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767816.902323:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967360384:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767816.902336:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767816.902342:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767816/real 1775767816] req@ffff8f994f2e4700 x1862022967360384/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767816.902349:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967360384/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767816.902355:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767816.902356:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767816.902358:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967360384:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767816.902363:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967360512:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767816.902373:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767816.902378:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767816/real 1775767816] req@ffff8f994f2e6300 x1862022967360512/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767816.902385:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967360512/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767816.902391:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767816.902392:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767816.902394:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967360512:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767817.797829:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767817.797876:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767818.821871:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767818.821982:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767819.845887:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767819.845968:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767820.869878:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767820.869957:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767821.893877:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767821.893953:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00080000:1.0:1775767822.021868:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767822.021877:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767822.021881:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767822.021891:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4443 00000100:00080000:1.0:1775767822.021903:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767822.021909:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767822.021956:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967360640/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767822.021980:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767822.021983:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767822.021984:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767822.021987:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4443 00000100:00080000:1.0:1775767822.021991:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767822.021994:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767822.022005:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767822.022022:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967360768/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767822.022026:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967360640:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775767822.022061:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767822.022065:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767822.022067:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767822.022073:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4443 00000100:00080000:1.0:1775767822.022077:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767822.022083:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767822.022089:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767822.022102:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967360896/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767822.022113:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1775767822.022121:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775767822.022125:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767822.022127:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767822.022131:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4443 00000100:00080000:1.0:1775767822.022137:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767822.022140:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767822.022145:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767822.022153:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967361024/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767822.022183:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767822/real 1775767822] req@ffff8f994f2e6300 x1862022967360640/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767822.022203:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967360640/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767822.022215:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767822.022219:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767822.022222:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967360640:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767822.022247:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967360768:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775767822.022265:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767822.022273:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767822/real 1775767822] req@ffff8f994f2e4700 x1862022967360768/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767822.022282:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967360768/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767822.022288:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767822.022290:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767822.022292:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967360768:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767822.022296:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967360896:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767822.022308:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767822.022313:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767822/real 1775767822] req@ffff8f994f2e7b80 x1862022967360896/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767822.022326:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967360896/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767822.022332:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767822.022333:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767822.022336:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967360896:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767822.022340:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967361024:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767822.022350:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767822.022356:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767822/real 1775767822] req@ffff8f994f2e4e00 x1862022967361024/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767822.022363:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967361024/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767822.022368:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767822.022386:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767822.022389:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967361024:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767822.917870:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767822.917965:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767823.941859:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767823.941878:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775767824.965895:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767824.965964:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767825.989838:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767825.989879:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767827.013859:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767827.013960:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767827.141849:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767827.141859:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767827.141864:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767827.141874:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4448 00000100:00080000:3.0:1775767827.141900:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767827.141908:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767827.141947:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967361152/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767827.141977:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767827.141981:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767827.141984:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767827.141990:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4448 00000100:00080000:3.0:1775767827.141995:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767827.142000:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767827.142005:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967361152:192.168.202.146@tcp:250:kworker.0 00000008:00100000:3.0:1775767827.142011:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767827.142028:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967361280/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767827.142041:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767827.142045:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767827.142047:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767827.142051:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4448 00000100:00080000:3.0:1775767827.142056:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767827.142060:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:2.0:1775767827.142065:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:3.0:1775767827.142067:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767827.142083:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967361408/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767827.142093:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767827.142097:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767827.142099:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767827.142103:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4448 00000100:00080000:3.0:1775767827.142107:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767827.142112:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767827.142118:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767827.142122:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767827/real 1775767827] req@ffff8f9970edfb80 x1862022967361152/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767827.142134:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967361536/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767827.142142:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967361152/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767827.142154:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767827.142158:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767827.142161:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967361152:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767827.142182:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967361280:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775767827.142199:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767827.142207:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767827/real 1775767827] req@ffff8f9970edc380 x1862022967361280/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767827.142215:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967361280/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767827.142221:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767827.142223:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767827.142225:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967361280:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767827.142228:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967361408:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767827.142238:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767827.142243:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767827/real 1775767827] req@ffff8f9970edca80 x1862022967361408/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767827.142250:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967361408/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767827.142260:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767827.142262:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767827.142264:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967361408:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767827.142267:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967361536:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767827.142276:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767827.142282:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767827/real 1775767827] req@ffff8f9970edea00 x1862022967361536/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767827.142288:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967361536/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767827.142294:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767827.142295:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767827.142297:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967361536:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767828.037809:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767828.037903:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767829.061894:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767829.061908:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767830.085846:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767830.085938:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767831.109818:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767831.109911:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767832.133867:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767832.133934:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767832.261969:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767832.261976:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767832.261980:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767832.261988:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4453 00000100:00080000:0.0:1775767832.261998:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767832.262003:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767832.262035:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967361664/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767832.262057:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767832.262060:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767832.262061:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767832.262064:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4453 00000100:00080000:0.0:1775767832.262066:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767832.262069:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767832.262076:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767832.262086:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967361792/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767832.262093:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967361664:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775767832.262097:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767832.262102:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767832.262103:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767832.262109:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4453 00000100:00080000:0.0:1775767832.262113:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767832.262117:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767832.262123:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767832.262137:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967361920/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767832.262148:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767832.262152:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767832.262156:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1775767832.262157:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775767832.262162:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4453 00000100:00080000:0.0:1775767832.262166:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767832.262170:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767832.262175:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767832.262185:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967362048/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767832.262200:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767832/real 1775767832] req@ffff8f9970a76a00 x1862022967361664/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767832.262215:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967361664/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767832.262223:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767832.262226:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767832.262229:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967361664:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767832.262249:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967361792:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775767832.262262:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767832.262270:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767832/real 1775767832] req@ffff8f9970a77100 x1862022967361792/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767832.262277:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967361792/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767832.262282:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767832.262284:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767832.262286:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967361792:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767832.262289:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967361920:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767832.262299:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767832.262305:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767832/real 1775767832] req@ffff8f9970a75c00 x1862022967361920/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767832.262315:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967361920/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767832.262324:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767832.262327:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767832.262330:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967361920:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767832.262336:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967362048:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767832.262357:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767832.262367:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767832/real 1775767832] req@ffff8f9970a77480 x1862022967362048/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767832.262378:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967362048/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767832.262386:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767832.262388:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767832.262390:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967362048:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767833.157834:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767833.157913:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767834.181875:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767834.181930:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767835.205866:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767835.205909:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767836.229931:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767836.229973:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767837.253834:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767837.253929:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767837.381866:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767837.381875:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767837.381879:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767837.381889:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4458 00000100:00080000:3.0:1775767837.381899:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767837.381906:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767837.381937:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd180 x1862022967362176/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767837.381964:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767837.381969:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767837.381972:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767837.381978:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4458 00000100:00080000:3.0:1775767837.381983:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767837.381989:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767837.381999:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967362176:192.168.202.146@tcp:250:kworker.0 00000008:00100000:3.0:1775767837.381999:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767837.382021:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddf80 x1862022967362304/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767837.382035:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767837.382039:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767837.382042:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767837.382048:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4458 00000100:00080000:3.0:1775767837.382071:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775767837.382077:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775767837.382079:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767837.382086:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767837.382122:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767837/real 1775767837] req@ffff8f9970edd180 x1862022967362176/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767837.382143:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd180 x1862022967362176/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767837.382154:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767837.382158:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767837.382162:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967362176:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767837.382187:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967362304:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775767837.382210:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767837.382220:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767837/real 1775767837] req@ffff8f9970eddf80 x1862022967362304/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767837.382240:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddf80 x1862022967362304/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767837.382246:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767837.382249:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767837.382253:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967362304:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767837.382281:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967362432/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767837.382295:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767837.382299:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767837.382302:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767837.382307:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4458 00000100:00080000:3.0:1775767837.382313:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767837.382319:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767837.382325:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767837.382327:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967362432:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767837.382360:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767837.382374:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767837/real 1775767837] req@ffff8f9970edea00 x1862022967362432/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767837.382385:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967362432/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767837.382391:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767837.382394:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767837.382397:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967362432:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767837.382406:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967362560/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767837.382439:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967362560:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767837.382462:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767837.382473:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767837/real 1775767837] req@ffff8f9970edca80 x1862022967362560/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767837.382487:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967362560/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767837.382495:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767837.382499:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767837.382502:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967362560:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767838.277882:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767838.277945:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767839.301819:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767839.301954:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767840.325807:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767840.325894:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767841.349811:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767841.349868:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767842.373836:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767842.373903:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767842.501874:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767842.501887:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767842.501893:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767842.501906:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4464 00000100:00080000:3.0:1775767842.501920:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767842.501928:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767842.501980:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967362688/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767842.502011:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767842.502015:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767842.502017:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767842.502022:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4464 00000100:00080000:3.0:1775767842.502025:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767842.502028:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767842.502039:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767842.502060:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967362816/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767842.502071:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967362688:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775767842.502075:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767842.502083:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767842.502087:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767842.502092:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4464 00000100:00080000:3.0:1775767842.502095:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767842.502098:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767842.502102:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767842.502117:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967362944/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767842.502126:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767842.502130:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767842.502131:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767842.502135:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4464 00000100:00080000:3.0:1775767842.502157:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767842.502160:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767842.502165:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767842.502176:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967363072/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767842.502293:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967362816:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767842.502510:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967362944:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767842.502567:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967363072:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767843.397930:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767843.397965:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767844.421886:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767844.421998:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767845.445821:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767845.445885:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1775767845.574020:0:2367:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:0.0:1775767845.574064:0:2367:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1775767845.574082:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775767845.574123:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:0.0:1775767845.574136:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:2.0:1775767845.574153:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767842/real 1775767845] req@ffff8f9970edfb80 x1862022967362816/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:0.0:1775767845.574154:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:2.0:1775767845.574183:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967362816/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767845.574197:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767845.574202:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767845.574207:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967362816:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767845.574215:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767842/real 1775767845] req@ffff8f9970ede300 x1862022967362944/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767845.574229:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967362944/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767845.574238:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767845.574242:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767845.574246:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967362944:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767845.574253:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767842/real 1775767845] req@ffff8f9970edd880 x1862022967363072/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767845.574266:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967363072/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767845.574275:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767845.574278:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767845.574282:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967363072:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767845.574288:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767842/real 1775767845] req@ffff8f9970edc380 x1862022967362688/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767845.574300:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967362688/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767845.574308:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767845.574313:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767845.574316:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967362688:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767846.469830:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767846.469900:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767847.493835:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767847.493955:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767847.621882:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767847.621891:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767847.621895:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767847.621903:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4469 00000100:00080000:1.0:1775767847.621914:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767847.621920:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767847.621953:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967363200/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767847.621983:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767847.621988:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767847.621990:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767847.621995:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4469 00000100:00080000:1.0:1775767847.622000:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767847.622005:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767847.622016:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767847.622043:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967363328/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767847.622058:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967363200:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775767847.622079:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767847.622082:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767847.622084:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767847.622089:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4469 00000100:00080000:1.0:1775767847.622093:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767847.622097:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767847.622103:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767847.622114:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967363456/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767847.622122:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767847.622125:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767847.622127:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767847.622129:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4469 00000100:00080000:1.0:1775767847.622132:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767847.622135:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767847.622138:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767847.622146:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967363584/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767847.622192:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767847.622283:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767847/real 1775767847] req@ffff8f994f2e6680 x1862022967363200/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767847.622311:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967363200/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767847.622326:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767847.622333:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767847.622338:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967363200:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767847.622390:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967363328:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775767847.622421:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767847.622437:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767847/real 1775767847] req@ffff8f994f2e4000 x1862022967363328/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767847.622450:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967363328/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767847.622458:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767847.622462:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767847.622466:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967363328:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767847.622476:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967363456:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767847.622512:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767847.622525:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767847/real 1775767847] req@ffff8f994f2e5f80 x1862022967363456/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767847.622537:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967363456/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767847.622547:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767847.622551:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767847.622555:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967363456:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767847.622561:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967363584:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767847.622584:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767847.622596:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767847/real 1775767847] req@ffff8f994f2e6a00 x1862022967363584/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767847.622609:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967363584/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767847.622618:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767847.622622:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767847.622626:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967363584:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767848.517868:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767848.517936:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767849.541791:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767849.541870:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767850.565876:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767850.565923:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767851.589844:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767851.589933:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767852.613807:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767852.613878:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767852.741830:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767852.741838:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767852.741842:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767852.741851:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4474 00000100:00080000:1.0:1775767852.741861:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767852.741866:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767852.741898:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967363712/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767852.741921:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767852.741924:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767852.741926:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767852.741929:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4474 00000100:00080000:1.0:1775767852.741931:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767852.741934:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767852.741942:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767852.741950:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967363712:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767852.741952:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967363840/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767852.741963:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767852.741965:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767852.741966:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767852.741969:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4474 00000100:00080000:1.0:1775767852.741971:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767852.741973:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767852.742003:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775767852.742011:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767852.742014:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967363968/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767852.742022:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767852.742024:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767852.742026:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767852.742028:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4474 00000100:00080000:1.0:1775767852.742031:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767852.742034:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767852.742037:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767852.742044:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967364096/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767852.742062:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767852/real 1775767852] req@ffff8f994f2e6d80 x1862022967363712/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767852.742079:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967363712/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767852.742087:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767852.742090:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767852.742093:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967363712:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767852.742130:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967363840:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775767852.742155:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767852.742167:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767852/real 1775767852] req@ffff8f994f2e4a80 x1862022967363840/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767852.742176:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967363840/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767852.742182:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767852.742184:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767852.742186:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967363840:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767852.742191:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967363968:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767852.742204:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767852.742209:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767852/real 1775767852] req@ffff8f994f2e7800 x1862022967363968/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767852.742217:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967363968/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767852.742222:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767852.742224:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767852.742226:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967363968:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767852.742230:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967364096:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767852.742239:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767852.742244:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767852/real 1775767852] req@ffff8f994f2e5180 x1862022967364096/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767852.742252:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967364096/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767852.742257:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767852.742258:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767852.742260:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967364096:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767853.637844:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767853.637915:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767854.661820:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767854.661867:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767855.685830:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767855.685897:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767856.709817:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767856.709925:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767857.733879:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767857.733913:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767857.861858:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767857.861868:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767857.861873:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767857.861885:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4479 00000100:00080000:3.0:1775767857.861898:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767857.861905:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767857.861950:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967364224/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767857.861977:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767857.861980:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767857.861982:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767857.861986:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4479 00000100:00080000:3.0:1775767857.861988:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767857.861992:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767857.862002:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767857.862012:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967364224:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767857.862013:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967364352/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767857.862026:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767857.862029:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767857.862031:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767857.862036:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4479 00000100:00080000:3.0:1775767857.862040:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767857.862045:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767857.862050:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767857.862073:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967364480/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767857.862086:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767857.862090:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767857.862092:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767857.862096:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4479 00000100:00080000:3.0:1775767857.862100:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767857.862103:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:2.0:1775767857.862107:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:3.0:1775767857.862108:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767857.862120:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967364608/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767857.862165:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767857/real 1775767857] req@ffff8f9970edf480 x1862022967364224/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767857.862185:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967364224/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767857.862194:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767857.862198:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767857.862201:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967364224:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767857.862224:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967364352:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775767857.862243:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767857.862251:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767857/real 1775767857] req@ffff8f9970edf800 x1862022967364352/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767857.862259:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967364352/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767857.862265:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767857.862268:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767857.862270:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967364352:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767857.862273:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967364480:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767857.862284:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767857.862289:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767857/real 1775767857] req@ffff8f9970eded80 x1862022967364480/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767857.862296:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967364480/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767857.862306:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767857.862308:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767857.862310:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967364480:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767857.862313:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967364608:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767857.862322:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767857.862327:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767857/real 1775767857] req@ffff8f9970ede680 x1862022967364608/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767857.862334:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967364608/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767857.862339:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767857.862341:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767857.862342:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967364608:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767858.757829:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767858.757913:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767859.781855:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767859.781922:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767860.805840:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767860.805947:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767861.829885:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767861.829951:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767862.853840:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767862.853924:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767862.981848:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767862.981856:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767862.981861:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767862.981870:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4484 00000100:00080000:0.0:1775767862.981880:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767862.981885:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767862.981917:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967364736/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767862.981940:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767862.981943:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767862.981944:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767862.981947:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4484 00000100:00080000:0.0:1775767862.981949:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767862.981951:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767862.981959:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767862.981968:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967364864/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767862.981976:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767862.981980:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1775767862.981981:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967364736:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775767862.981982:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767862.981990:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4484 00000100:00080000:0.0:1775767862.981992:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767862.981994:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767862.982035:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775767862.982041:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767862.982047:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967364992/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767862.982054:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767862.982057:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767862.982058:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767862.982061:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4484 00000100:00080000:0.0:1775767862.982064:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767862.982066:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767862.982070:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767862.982078:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967365120/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767862.982085:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767862/real 1775767862] req@ffff8f9970a75500 x1862022967364736/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767862.982103:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967364736/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767862.982113:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767862.982116:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767862.982120:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967364736:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767862.982142:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967364864:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775767862.982165:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767862.982177:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767862/real 1775767862] req@ffff8f9970a76300 x1862022967364864/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767862.982191:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967364864/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767862.982199:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767862.982203:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767862.982206:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967364864:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775767862.982213:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967364992:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767862.982233:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767862.982243:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767862/real 1775767862] req@ffff8f9970a75180 x1862022967364992/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767862.982256:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967364992/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767862.982264:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767862.982267:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767862.982271:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967364992:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767862.982277:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967365120:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775767862.982296:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767862.982306:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767862/real 1775767862] req@ffff8f9970a77480 x1862022967365120/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767862.982315:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967365120/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767862.982320:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767862.982323:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775767862.982325:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967365120:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767863.877860:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767863.877920:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767864.901902:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767864.901937:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767865.925840:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767865.925902:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767866.949803:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767866.949915:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767867.973827:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767867.973922:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767868.102091:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767868.102104:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767868.102111:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767868.102125:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4489 00000100:00080000:3.0:1775767868.102141:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767868.102149:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767868.102193:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967365248/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767868.102226:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767868.102230:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767868.102233:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767868.102237:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4489 00000100:00080000:3.0:1775767868.102241:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767868.102245:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767868.102256:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767868.102268:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967365376/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767868.102277:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767868.102281:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767868.102283:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767868.102288:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4489 00000100:00100000:1.0:1775767868.102294:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967365248:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775767868.102299:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767868.102306:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767868.102311:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767868.102326:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf100 x1862022967365504/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767868.102345:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767868.102349:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767868.102350:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767868.102355:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4489 00000100:00080000:3.0:1775767868.102360:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767868.102363:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:1.0:1775767868.102369:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:3.0:1775767868.102369:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767868.102386:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967365632/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767868.102437:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767868/real 1775767868] req@ffff8f9970edc700 x1862022967365248/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767868.102458:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967365248/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767868.102469:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767868.102473:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767868.102478:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967365248:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767868.102503:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967365376:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775767868.102529:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767868.102567:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767868/real 1775767868] req@ffff8f9970edc000 x1862022967365376/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767868.102587:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967365376/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767868.102597:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767868.102602:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767868.102605:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967365376:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767868.102635:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967365504:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767868.102660:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767868.102672:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767868/real 1775767868] req@ffff8f9970edf100 x1862022967365504/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767868.102687:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf100 x1862022967365504/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767868.102718:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767868.102722:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767868.102725:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967365504:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767868.102732:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967365632:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767868.102801:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767868.102827:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767868/real 1775767868] req@ffff8f9970edd500 x1862022967365632/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767868.102839:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967365632/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767868.102848:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767868.102852:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767868.102856:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967365632:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767868.997854:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767868.997928:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767870.021876:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767870.021881:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767871.045898:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767871.045973:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767872.069917:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767872.069938:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775767873.093876:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767873.093965:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767873.221835:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767873.221846:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767873.221852:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767873.221874:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4494 00000100:00080000:3.0:1775767873.221886:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767873.221895:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767873.221934:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddc00 x1862022967365760/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767873.221961:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767873.221965:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767873.221967:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767873.221975:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4494 00000100:00080000:3.0:1775767873.221979:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767873.221984:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767873.221993:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967365760:192.168.202.146@tcp:250:kworker.0 00000008:00100000:3.0:1775767873.221994:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767873.222013:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967365888/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767873.222031:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767873.222035:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767873.222037:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767873.222045:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4494 00000100:00080000:3.0:1775767873.222051:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1775767873.222057:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775767873.222057:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767873.222065:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767873.222079:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967366016/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767873.222090:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775767873.222094:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767873/real 1775767873] req@ffff8f9970eddc00 x1862022967365760/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767873.222095:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767873.222099:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775767873.222114:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddc00 x1862022967365760/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767873.222126:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767873.222129:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767873.222133:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967365760:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767873.222151:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967365888:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775767873.222166:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767873.222175:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767873/real 1775767873] req@ffff8f9970edce00 x1862022967365888/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767873.222185:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967365888/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767873.222192:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4494 00000100:00080000:1.0:1775767873.222197:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767873.222199:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767873.222203:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00080000:3.0:1775767873.222206:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767873.222210:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967365888:192.168.202.146@tcp:38:kworker.0 00000008:00100000:3.0:1775767873.222214:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767873.222217:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967366016:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767873.222234:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767873.222241:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767873/real 1775767873] req@ffff8f9970edd500 x1862022967366016/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767873.222248:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967366016/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767873.222254:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767873.222256:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767873.222258:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967366016:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767873.222338:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf100 x1862022967366144/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767873.222394:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967366144:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767873.222420:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767873.222430:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767873/real 1775767873] req@ffff8f9970edf100 x1862022967366144/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767873.222459:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf100 x1862022967366144/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767873.222476:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767873.222480:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767873.222486:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967366144:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767874.117850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767874.117903:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767875.141873:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767875.141945:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767876.165868:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767876.165937:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767877.189865:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767877.190013:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767878.213881:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767878.213952:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767878.341865:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767878.341876:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767878.341882:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767878.341895:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4499 00000100:00080000:0.0:1775767878.341910:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767878.341917:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767878.341968:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967366272/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767878.342002:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775767878.342070:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967366272:192.168.202.146@tcp:250:kworker.0 00000800:00000100:1.0:1775767878.342154:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775767878.342183:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767878.342188:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767878.342199:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4499 00000100:00100000:1.0:1775767878.342205:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767878/real 1775767878] req@ffff8f9970a75c00 x1862022967366272/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767878.342214:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767878.342223:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767878.342226:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967366272/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767878.342237:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1775767878.342238:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1775767878.342242:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767878.342249:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967366272:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767878.342282:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967366400/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767878.342317:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767878.342320:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767878.342322:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767878.342325:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4499 00000100:00080000:0.0:1775767878.342329:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767878.342332:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767878.342338:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767878.342351:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967366528/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767878.342356:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967366400:192.168.202.146@tcp:38:kworker.0 00000100:00080000:0.0:1775767878.342380:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767878.342383:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767878.342397:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767878.342402:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4499 00000100:00080000:0.0:1775767878.342412:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767878.342418:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:1.0:1775767878.342418:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767878.342435:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767878/real 1775767878] req@ffff8f9970a77100 x1862022967366400/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1775767878.342445:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767878.342460:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967366400/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767878.342466:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967366656/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767878.342489:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767878.342497:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767878.342505:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967366400:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767878.342525:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967366528:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767878.342543:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767878.342553:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767878/real 1775767878] req@ffff8f9970a76a00 x1862022967366528/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767878.342562:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967366528/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767878.342568:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767878.342570:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767878.342572:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967366528:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767878.342577:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967366656:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767878.342587:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767878.342593:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767878/real 1775767878] req@ffff8f9970a74380 x1862022967366656/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767878.342601:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967366656/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767878.342607:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767878.342609:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767878.342611:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967366656:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775767879.237945:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767879.237948:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775767880.261825:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767880.261922:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767881.285869:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767881.285938:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767882.309867:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767882.309880:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767883.333889:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767883.333922:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767883.461903:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767883.461911:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767883.461916:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767883.461926:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4505 00000100:00080000:3.0:1775767883.461936:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767883.461941:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767883.461973:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967366784/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767883.462000:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767883.462003:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767883.462004:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767883.462008:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4505 00000100:00080000:3.0:1775767883.462010:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767883.462013:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767883.462020:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767883.462028:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967366912/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767883.462035:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767883.462038:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767883.462039:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767883.462041:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4505 00000100:00080000:3.0:1775767883.462043:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767883.462046:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767883.462049:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767883.462059:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967367040/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767883.462061:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967366784:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775767883.462074:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767883.462076:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767883.462077:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767883.462079:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4505 00000100:00080000:3.0:1775767883.462081:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767883.462083:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767883.462086:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767883.462094:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967367168/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767883.462127:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767883.462173:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767883/real 1775767883] req@ffff8f9970edc000 x1862022967366784/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767883.462193:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967366784/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767883.462201:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767883.462204:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767883.462207:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967366784:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767883.462213:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967366912:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775767883.462231:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767883.462242:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767883/real 1775767883] req@ffff8f9970edc700 x1862022967366912/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767883.462261:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967366912/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767883.462270:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767883.462274:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767883.462278:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967366912:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767883.462301:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967367040:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767883.462318:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767883.462329:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767883/real 1775767883] req@ffff8f9970ede680 x1862022967367040/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767883.462372:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967367040/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767883.462382:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767883.462385:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767883.462389:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967367040:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767883.462395:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967367168:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767883.462411:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767883.462418:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767883/real 1775767883] req@ffff8f9970eded80 x1862022967367168/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767883.462426:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967367168/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767883.462435:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767883.462437:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767883.462439:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967367168:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767884.357860:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767884.357911:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767885.381825:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767885.381923:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767886.405910:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767886.405917:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775767887.429819:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767887.429903:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767888.453849:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767888.453889:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767888.581860:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767888.581871:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767888.581876:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767888.581887:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4510 00000100:00080000:0.0:1775767888.581898:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767888.581906:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767888.581938:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967367296/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767888.581964:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767888.581968:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767888.581970:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767888.581975:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4510 00000100:00080000:0.0:1775767888.581979:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767888.581983:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767888.581993:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767888.582008:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967367424/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767888.582020:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767888.582023:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767888.582026:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775767888.582029:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967367296:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775767888.582030:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4510 00000100:00080000:0.0:1775767888.582035:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767888.582041:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767888.582045:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767888.582055:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967367552/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767888.582070:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767888.582073:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767888.582074:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767888.582077:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4510 00000100:00080000:0.0:1775767888.582091:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767888.582093:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767888.582097:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775767888.582103:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767888.582106:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967367680/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767888.582166:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767888/real 1775767888] req@ffff8f9970a77800 x1862022967367296/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767888.582187:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967367296/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767888.582198:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767888.582203:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767888.582208:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967367296:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767888.582232:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967367424:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775767888.582268:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767888.582283:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767888/real 1775767888] req@ffff8f9970a76d80 x1862022967367424/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767888.582333:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967367424/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767888.582378:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767888.582382:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767888.582387:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967367424:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767888.582393:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967367552:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767888.582416:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767888.582428:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767888/real 1775767888] req@ffff8f9970a74700 x1862022967367552/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767888.582440:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967367552/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767888.582449:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767888.582452:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767888.582456:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967367552:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767888.582462:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967367680:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767888.582481:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767888.582493:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767888/real 1775767888] req@ffff8f9970a74380 x1862022967367680/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767888.582505:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967367680/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767888.582513:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767888.582517:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767888.582521:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967367680:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767889.477844:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767889.477848:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767890.501849:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767890.501934:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767891.525846:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767891.525906:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767892.549842:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767892.549921:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767893.573875:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767893.573879:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00080000:3.0:1775767893.701876:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767893.701888:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767893.701893:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767893.701903:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4515 00000100:00080000:3.0:1775767893.701915:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767893.701921:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767893.701955:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967367808/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767893.701985:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767893.701990:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767893.701992:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767893.701996:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4515 00000100:00080000:3.0:1775767893.702001:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767893.702006:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767893.702015:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767893.702026:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967367808:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767893.702031:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967367936/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767893.702041:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767893.702043:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767893.702044:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767893.702047:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4515 00000100:00080000:3.0:1775767893.702050:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767893.702054:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767893.702058:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767893.702066:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967368064/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767893.702072:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767893.702075:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767893.702076:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767893.702078:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4515 00000100:00080000:3.0:1775767893.702081:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1775767893.702085:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775767893.702085:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767893.702089:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767893.702097:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967368192/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767893.702136:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767893/real 1775767893] req@ffff8f9970edf800 x1862022967367808/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767893.702150:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967367808/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767893.702158:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767893.702161:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767893.702164:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967367808:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767893.702183:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967367936:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775767893.702198:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767893.702206:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767893/real 1775767893] req@ffff8f9970edf480 x1862022967367936/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767893.702214:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967367936/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767893.702220:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767893.702222:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767893.702224:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967367936:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767893.702287:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967368064:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767893.702303:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767893.702311:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767893/real 1775767893] req@ffff8f9970edc380 x1862022967368064/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767893.702319:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967368064/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767893.702325:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767893.702327:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767893.702329:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967368064:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767893.702333:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967368192:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767893.702343:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767893.702349:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767893/real 1775767893] req@ffff8f9970edd880 x1862022967368192/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767893.702356:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967368192/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767893.702361:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767893.702363:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767893.702365:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967368192:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767894.597840:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767894.597894:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767895.621869:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767895.621998:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767896.645851:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767896.645936:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767897.669829:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767897.669906:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767898.693865:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767898.693935:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767898.821814:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767898.821821:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767898.821825:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767898.821833:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4520 00000100:00080000:3.0:1775767898.821844:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767898.821850:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767898.821881:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967368320/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767898.821904:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767898.821907:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767898.821933:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767898.821938:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4520 00000100:00080000:3.0:1775767898.821941:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767898.821945:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767898.821953:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767898.821957:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967368320:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767898.821970:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967368448/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767898.821985:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767898.821989:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767898.821992:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767898.821996:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4520 00000100:00080000:3.0:1775767898.822001:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767898.822005:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767898.822011:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775767898.822018:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767898.822029:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967368576/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767898.822041:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767898.822045:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767898.822047:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767898.822051:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4520 00000100:00080000:3.0:1775767898.822056:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767898.822061:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767898.822067:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767898.822069:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767898/real 1775767898] req@ffff8f9970ede300 x1862022967368320/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767898.822095:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967368320/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767898.822102:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967368704/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767898.822116:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767898.822125:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767898.822131:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967368320:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767898.822156:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967368448:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775767898.822179:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767898.822216:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767898/real 1775767898] req@ffff8f9970edfb80 x1862022967368448/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767898.822226:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967368448/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767898.822232:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767898.822234:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767898.822236:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967368448:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767898.822241:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967368576:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767898.822256:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767898.822262:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767898/real 1775767898] req@ffff8f9970edca80 x1862022967368576/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767898.822292:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967368576/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767898.822297:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767898.822299:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767898.822301:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967368576:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767898.822305:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967368704:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767898.822317:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767898.822324:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767898/real 1775767898] req@ffff8f9970edea00 x1862022967368704/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767898.822331:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967368704/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767898.822336:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767898.822337:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767898.822339:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967368704:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767899.717812:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767899.717890:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767900.741889:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767900.741934:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767901.765867:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767901.765939:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767902.789903:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767902.789954:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767903.813861:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767903.813906:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767903.941874:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767903.941889:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767903.941895:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767903.941913:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4525 00000100:00080000:3.0:1775767903.941928:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767903.941936:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767903.941985:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddf80 x1862022967368832/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767903.942016:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767903.942020:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767903.942021:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767903.942025:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4525 00000100:00080000:3.0:1775767903.942028:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767903.942031:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767903.942042:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767903.942058:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd180 x1862022967368960/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767903.942077:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967368832:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775767903.942090:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767903.942095:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767903.942098:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767903.942103:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4525 00000100:00080000:3.0:1775767903.942107:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1775767903.942188:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767903.942244:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767903/real 1775767903] req@ffff8f9970eddf80 x1862022967368832/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767903.942266:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddf80 x1862022967368832/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767903.942275:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767903.942278:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767903.942281:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967368832:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767903.942303:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967368960:192.168.202.146@tcp:38:kworker.0 00000100:00080000:3.0:1775767903.942307:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767903.942319:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775767903.942326:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767903.942336:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767903/real 1775767903] req@ffff8f9970edd180 x1862022967368960/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767903.942351:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd180 x1862022967368960/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767903.942353:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddf80 x1862022967369088/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767903.942384:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767903.942389:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767903.942391:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00080000:3.0:1775767903.942394:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767903.942397:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775767903.942398:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967368960:192.168.202.146@tcp:38:kworker.0 00000100:00080000:3.0:1775767903.942404:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4525 00000100:00080000:3.0:1775767903.942410:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767903.942413:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767903.942417:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767903.942420:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967369088:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767903.942428:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967369216/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767903.942449:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767903.942464:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767903/real 1775767903] req@ffff8f9970eddf80 x1862022967369088/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767903.942475:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddf80 x1862022967369088/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767903.942489:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767903.942493:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767903.942497:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967369088:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767903.942509:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967369216:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767903.942526:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767903.942536:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767903/real 1775767903] req@ffff8f9970edea00 x1862022967369216/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767903.942547:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967369216/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767903.942555:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767903.942559:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767903.942562:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967369216:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767904.837852:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767904.837919:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767905.861851:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767905.861898:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767906.885881:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767906.885948:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767907.909891:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767907.909945:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767908.933834:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767908.933889:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767909.061816:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767909.061824:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767909.061828:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767909.061836:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4530 00000100:00080000:0.0:1775767909.061846:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767909.061889:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767909.061921:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967369344/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767909.061945:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767909.061948:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767909.061949:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767909.061954:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4530 00000100:00080000:0.0:1775767909.061957:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767909.061961:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767909.061970:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767909.061982:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967369344:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767909.061984:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967369472/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767909.061998:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767909.062001:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767909.062002:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767909.062005:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4530 00000100:00080000:0.0:1775767909.062008:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767909.062011:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767909.062015:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767909.062026:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967369600/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767909.062032:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767909.062035:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767909.062047:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767909.062050:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4530 00000100:00080000:0.0:1775767909.062053:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767909.062055:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775767909.062059:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767909.062067:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967369728/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767909.062270:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967369472:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767909.062330:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967369600:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767909.062360:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967369728:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767909.957823:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767909.957877:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767910.981825:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767910.981938:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767912.005862:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767912.005958:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775767912.134168:0:2369:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:3.0:1775767912.134207:0:2369:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775767912.134225:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:3.0:1775767912.134270:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:1.0:1775767912.134320:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767909/real 1775767912] req@ffff8f9970a76a00 x1862022967369344/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1775767912.134331:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:3.0:1775767912.134346:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:1.0:1775767912.134360:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967369344/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767912.134400:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767912.134406:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767912.134410:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967369344:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767912.134437:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767909/real 1775767912] req@ffff8f9970a77100 x1862022967369472/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767912.134451:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967369472/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767912.134460:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767912.134463:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767912.134467:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967369472:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767912.134474:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767909/real 1775767912] req@ffff8f9970a75c00 x1862022967369600/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767912.134485:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967369600/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767912.134493:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767912.134497:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767912.134500:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967369600:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767912.134506:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767909/real 1775767912] req@ffff8f9970a77480 x1862022967369728/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767912.134518:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967369728/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767912.134541:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767912.134544:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767912.134548:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967369728:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767913.029828:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767913.029911:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767914.053846:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767914.053912:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767914.181889:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767914.181897:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767914.181901:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767914.181910:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4535 00000100:00080000:1.0:1775767914.181919:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767914.181925:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767914.181955:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967369856/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767914.181985:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767914.181989:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767914.181991:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767914.181996:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4535 00000100:00080000:1.0:1775767914.182000:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767914.182004:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767914.182014:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767914.182028:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967369984/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767914.182055:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767914.182059:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767914.182061:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767914.182066:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4535 00000100:00080000:1.0:1775767914.182070:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775767914.182075:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967369856:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775767914.182078:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767914.182087:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767914.182101:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967370112/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767914.182113:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767914.182117:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767914.182118:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767914.182122:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4535 00000100:00080000:1.0:1775767914.182126:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767914.182130:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767914.182135:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767914.182148:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967370240/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775767914.182153:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767914.182219:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767914/real 1775767914] req@ffff8f994f2e5180 x1862022967369856/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767914.182242:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967369856/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767914.182253:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767914.182258:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767914.182262:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967369856:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767914.182270:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967369984:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767914.182306:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767914.182320:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767914/real 1775767914] req@ffff8f994f2e7800 x1862022967369984/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767914.182331:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967369984/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767914.182339:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767914.182342:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767914.182345:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967369984:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767914.182369:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967370112:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767914.182390:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767914.182450:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767914/real 1775767914] req@ffff8f994f2e4a80 x1862022967370112/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767914.182463:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967370112/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767914.182472:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767914.182475:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767914.182479:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967370112:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767914.182485:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967370240:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767914.182509:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767914.182521:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767914/real 1775767914] req@ffff8f994f2e6d80 x1862022967370240/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767914.182533:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967370240/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767914.182540:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767914.182544:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767914.182547:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967370240:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767915.077871:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767915.077892:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767916.101864:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767916.101915:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767917.125847:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767917.125921:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767918.149861:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767918.149927:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767919.173856:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767919.173913:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767919.301831:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767919.301840:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767919.301845:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767919.301855:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4540 00000100:00080000:1.0:1775767919.301866:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767919.301873:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767919.301919:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967370368/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767919.301945:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767919.301948:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767919.301950:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767919.301955:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4540 00000100:00080000:1.0:1775767919.301960:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767919.301965:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767919.301975:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767919.301986:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967370368:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767919.302005:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967370496/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767919.302018:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767919.302022:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767919.302024:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767919.302028:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4540 00000100:00080000:1.0:1775767919.302033:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767919.302037:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767919.302043:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767919.302057:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967370624/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775767919.302069:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775767919.302081:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767919.302085:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767919.302088:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767919.302093:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4540 00000100:00080000:1.0:1775767919.302098:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767919.302102:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767919.302108:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767919.302111:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767919/real 1775767919] req@ffff8f994f2e6a00 x1862022967370368/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767919.302123:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967370752/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767919.302135:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967370368/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767919.302187:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767919.302192:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767919.302197:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967370368:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767919.302221:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967370496:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767919.302244:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767919.302256:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767919/real 1775767919] req@ffff8f994f2e5f80 x1862022967370496/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767919.302269:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967370496/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767919.302278:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767919.302282:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767919.302285:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967370496:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767919.302291:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967370624:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767919.302310:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767919.302321:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767919/real 1775767919] req@ffff8f994f2e4000 x1862022967370624/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767919.302334:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967370624/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767919.302342:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767919.302344:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767919.302347:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967370624:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767919.302365:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967370752:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767919.302381:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767919.302389:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767919/real 1775767919] req@ffff8f994f2e6680 x1862022967370752/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767919.302397:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967370752/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767919.302402:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767919.302405:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767919.302407:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967370752:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767920.197847:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767920.197957:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767921.221832:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767921.221892:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767922.245910:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767922.245932:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767923.269853:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767923.269915:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767924.293873:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767924.293935:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767924.421929:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767924.421957:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767924.421963:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767924.421977:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4546 00000100:00080000:3.0:1775767924.421993:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767924.422002:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767924.422050:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967370880/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767924.422096:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767924.422107:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767924.422110:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767924.422118:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4546 00000100:00080000:3.0:1775767924.422123:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775767924.422152:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967370880:192.168.202.146@tcp:250:kworker.0 00000800:00000100:1.0:1775767924.422238:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767924.422301:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767924/real 1775767924] req@ffff8f9970edca80 x1862022967370880/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767924.422311:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767924.422331:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767924.422334:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967370880/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767924.422352:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767924.422358:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767924.422363:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967370880:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767924.422374:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967371008/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767924.422394:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775767924.422400:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967371008:192.168.202.146@tcp:38:kworker.0 00000100:00080000:3.0:1775767924.422402:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767924.422409:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767924.422417:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4546 00000100:00080000:3.0:1775767924.422429:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767924.422439:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:1.0:1775767924.422441:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:3.0:1775767924.422450:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767924.422464:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767924/real 1775767924] req@ffff8f9970edfb80 x1862022967371008/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767924.422471:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967371136/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767924.422499:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967371008/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767924.422503:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767924.422522:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767924.422527:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767924.422529:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767924.422535:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4546 00000100:00080000:1.0:1775767924.422540:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00080000:3.0:1775767924.422540:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767924.422555:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767924.422556:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967371008:192.168.202.146@tcp:38:kworker.0 00000008:00100000:3.0:1775767924.422566:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767924.422585:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967371136:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767924.422585:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967371264/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767924.422627:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767924.422643:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767924/real 1775767924] req@ffff8f9970ede300 x1862022967371136/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767924.422658:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967371136/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767924.422667:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767924.422671:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767924.422675:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967371136:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767924.422689:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967371264:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767924.422725:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767924.422762:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767924/real 1775767924] req@ffff8f9970edd880 x1862022967371264/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767924.422934:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967371264/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767924.422953:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767924.422957:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767924.422960:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967371264:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767925.317886:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767925.317957:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767926.341838:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767926.341936:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767927.365858:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767927.365892:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767928.389828:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767928.389885:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767929.413962:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767929.413968:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00080000:3.0:1775767929.541819:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767929.541852:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767929.541858:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767929.541868:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4551 00000100:00080000:3.0:1775767929.541880:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767929.541887:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767929.541951:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967371392/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767929.541975:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767929.541978:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767929.541980:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767929.541984:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4551 00000100:00080000:3.0:1775767929.541987:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767929.541989:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767929.541999:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767929.542008:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967371520/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767929.542015:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775767929.542020:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967371392:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775767929.542020:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767929.542021:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767929.542028:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4551 00000100:00080000:3.0:1775767929.542031:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767929.542033:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767929.542036:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767929.542046:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967371648/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767929.542065:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767929.542067:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767929.542068:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767929.542072:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4551 00000100:00080000:3.0:1775767929.542075:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767929.542077:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767929.542080:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775767929.542085:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767929.542090:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967371776/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767929.542132:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767929/real 1775767929] req@ffff8f9970edc380 x1862022967371392/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767929.542152:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967371392/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767929.542164:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767929.542168:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767929.542171:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967371392:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775767929.542190:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967371520:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775767929.542208:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767929.542217:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767929/real 1775767929] req@ffff8f9970edf480 x1862022967371520/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767929.542256:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967371520/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767929.542265:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767929.542268:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767929.542272:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967371520:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775767929.542278:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967371648:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767929.542300:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767929.542311:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767929/real 1775767929] req@ffff8f9970edf800 x1862022967371648/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767929.542323:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967371648/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767929.542329:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767929.542331:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767929.542333:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967371648:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775767929.542337:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967371776:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775767929.542350:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775767929.542356:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767929/real 1775767929] req@ffff8f9970eded80 x1862022967371776/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767929.542363:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967371776/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767929.542386:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767929.542389:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775767929.542391:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967371776:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767930.437874:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767930.437955:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767931.461886:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767931.461936:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767932.485859:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767932.486001:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767933.509875:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767933.509914:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767934.533905:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767934.533966:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767934.662378:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767934.662397:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767934.662404:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767934.662420:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4556 00000100:00080000:2.0:1775767934.662436:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767934.662446:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767934.662497:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3480 x1862022967371904/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767934.662534:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767934.662544:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767934.662548:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767934.662554:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4556 00000100:00080000:2.0:1775767934.662559:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767934.662564:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767934.662577:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767934.662582:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967371904:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767934.662595:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1180 x1862022967372032/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767934.662607:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767934.662611:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767934.662613:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767934.662618:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4556 00000100:00080000:2.0:1775767934.662622:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767934.662627:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767934.662633:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767934.662646:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2300 x1862022967372160/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767934.662657:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767934.662661:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767934.662664:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767934.662668:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4556 00000100:00080000:2.0:1775767934.662672:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1775767934.662678:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775767934.662679:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767934.662687:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767934.662701:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3100 x1862022967372288/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767934.662785:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767934/real 1775767934] req@ffff8f99703d3480 x1862022967371904/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767934.662814:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3480 x1862022967371904/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767934.662837:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767934.662859:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767934.662865:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967371904:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767934.662895:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967372032:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767934.662917:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767934.662927:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767934/real 1775767934] req@ffff8f99703d1180 x1862022967372032/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767934.663150:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1180 x1862022967372032/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767934.663164:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767934.663170:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767934.663175:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967372032:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767934.663189:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967372160:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767934.663258:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767934.663295:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767934/real 1775767934] req@ffff8f99703d2300 x1862022967372160/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767934.663309:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2300 x1862022967372160/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767934.663325:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767934.663329:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767934.663332:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967372160:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767934.663340:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967372288:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767934.663364:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767934.663376:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767934/real 1775767934] req@ffff8f99703d3100 x1862022967372288/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767934.663387:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3100 x1862022967372288/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767934.663422:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767934.663426:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767934.663430:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967372288:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767935.557886:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767935.557955:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767936.581939:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767936.581974:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775767937.605904:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767937.605933:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767938.629866:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767938.629878:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767939.653904:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767939.653983:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767939.781891:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767939.781901:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767939.781907:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767939.781919:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4561 00000100:00080000:2.0:1775767939.781934:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767939.781942:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767939.781996:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2d80 x1862022967372416/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767939.782031:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767939.782036:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767939.782038:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767939.782043:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4561 00000100:00080000:2.0:1775767939.782047:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767939.782052:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767939.782065:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767939.782092:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967372416:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767939.782094:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1500 x1862022967372544/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767939.782121:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767939.782125:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767939.782128:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767939.782134:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4561 00000100:00080000:2.0:1775767939.782139:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767939.782144:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767939.782153:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767939.782170:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0700 x1862022967372672/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767939.782182:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767939.782186:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767939.782191:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1775767939.782191:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775767939.782215:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4561 00000100:00080000:2.0:1775767939.782223:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767939.782228:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767939.782235:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767939.782249:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767939/real 1775767939] req@ffff8f99703d2d80 x1862022967372416/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767939.782254:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0e00 x1862022967372800/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767939.782334:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2d80 x1862022967372416/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767939.782346:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767939.782350:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767939.782359:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967372416:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767939.782387:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967372544:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775767939.782416:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767939.782435:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767939/real 1775767939] req@ffff8f99703d1500 x1862022967372544/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767939.782450:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1500 x1862022967372544/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767939.782460:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767939.782464:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767939.782468:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967372544:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775767939.782515:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967372672:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767939.782534:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767939.782543:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767939/real 1775767939] req@ffff8f99703d0700 x1862022967372672/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767939.782551:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0700 x1862022967372672/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767939.782557:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767939.782559:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767939.782561:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967372672:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775767939.782565:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967372800:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775767939.782577:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775767939.782582:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767939/real 1775767939] req@ffff8f99703d0e00 x1862022967372800/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767939.782590:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0e00 x1862022967372800/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767939.782595:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767939.782597:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775767939.782599:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967372800:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775767940.677882:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767940.677890:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775767941.701886:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767941.701915:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767942.725836:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767942.725889:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767943.749904:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767943.749927:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775767944.773866:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767944.773902:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767944.901936:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767944.901947:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767944.901952:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767944.902072:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4566 00000100:00080000:3.0:1775767944.902086:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767944.902097:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767944.902389:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967372928/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767944.902417:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767944.902420:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767944.902421:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767944.902425:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4566 00000100:00080000:3.0:1775767944.902428:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767944.902431:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767944.902443:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767944.902454:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967373056/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767944.902477:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967372928:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775767944.902514:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767944.902519:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767944.902521:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767944.902528:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4566 00000100:00080000:3.0:1775767944.902533:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767944.902538:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:0.0:1775767944.902544:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:3.0:1775767944.902546:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767944.902565:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967373184/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767944.902580:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767944/real 1775767944] req@ffff8f9970ede680 x1862022967372928/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767944.902595:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967372928/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767944.902603:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767944.902607:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767944.902609:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967372928:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775767944.902628:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767944.902631:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1775767944.902634:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967373056:192.168.202.146@tcp:38:kworker.0 00000100:00080000:3.0:1775767944.902637:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767944.902654:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4566 00000100:00080000:3.0:1775767944.902665:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775767944.902669:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775767944.902673:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767944.902682:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767944/real 1775767944] req@ffff8f9970edc700 x1862022967373056/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1775767944.902683:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767944.902699:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967373056/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767944.902707:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1775767944.902711:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf100 x1862022967373312/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767944.902713:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767944.902720:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967373056:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767944.902736:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967373184:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767944.902807:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767944.902833:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767944/real 1775767944] req@ffff8f9970edc000 x1862022967373184/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767944.902844:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967373184/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767944.902850:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767944.902853:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767944.902855:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967373184:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767944.902867:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967373312:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767944.902878:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767944.902884:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767944/real 1775767944] req@ffff8f9970edf100 x1862022967373312/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767944.902897:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf100 x1862022967373312/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767944.902903:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767944.902905:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767944.902907:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967373312:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767945.797882:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767945.797936:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767946.821851:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767946.821950:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767947.845835:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767947.845936:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767948.869892:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767948.869969:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767949.893852:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767949.893925:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767950.021887:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767950.021902:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767950.021907:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767950.021919:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4571 00000100:00080000:3.0:1775767950.021932:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767950.021940:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767950.021984:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967373440/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767950.022013:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767950.022017:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767950.022019:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767950.022024:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4571 00000100:00080000:3.0:1775767950.022027:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767950.022032:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767950.022045:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767950.022061:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967373440:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767950.022062:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967373568/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767950.022078:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767950.022080:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767950.022082:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767950.022086:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4571 00000100:00080000:3.0:1775767950.022089:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767950.022092:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767950.022097:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767950.022114:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddc00 x1862022967373696/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767950.022125:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767950.022129:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767950.022131:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767950.022135:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4571 00000800:00000100:0.0:1775767950.022139:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775767950.022147:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767950.022153:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767950.022160:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767950.022172:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf100 x1862022967373824/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767950.022185:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767950/real 1775767950] req@ffff8f9970edd500 x1862022967373440/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767950.022202:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967373440/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767950.022212:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767950.022216:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767950.022220:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967373440:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767950.022242:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967373568:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775767950.022257:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767950.022265:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767950/real 1775767950] req@ffff8f9970edce00 x1862022967373568/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767950.022273:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967373568/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767950.022466:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767950.022474:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767950.022479:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967373568:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767950.022490:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967373696:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767950.022553:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767950.022576:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767950/real 1775767950] req@ffff8f9970eddc00 x1862022967373696/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767950.022588:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddc00 x1862022967373696/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767950.022594:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767950.022596:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767950.022598:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967373696:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767950.022603:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967373824:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767950.022614:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767950.022620:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767950/real 1775767950] req@ffff8f9970edf100 x1862022967373824/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767950.022628:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf100 x1862022967373824/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767950.022636:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767950.022640:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767950.022644:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967373824:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767950.917910:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767950.917936:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767951.941822:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767951.941898:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767952.965857:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767952.965924:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767953.989794:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767953.989887:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767955.013872:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767955.013966:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767955.141890:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767955.141898:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767955.141902:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767955.141929:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4576 00000100:00080000:1.0:1775767955.141941:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767955.141974:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775767955.142006:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967373952/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767955.142031:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767955.142034:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767955.142035:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767955.142038:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4576 00000100:00080000:1.0:1775767955.142040:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767955.142043:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767955.142050:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767955.142060:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967374080/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767955.142071:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767955.142075:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767955.142077:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767955.142084:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4576 00000100:00080000:1.0:1775767955.142087:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767955.142109:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767955.142112:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967373952:192.168.202.146@tcp:250:kworker.0 00000008:00100000:1.0:1775767955.142119:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767955.142143:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967374208/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767955.142152:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767955.142154:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767955.142156:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767955.142159:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4576 00000100:00080000:1.0:1775767955.142162:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767955.142166:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775767955.142171:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767955.142181:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967374336/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767955.142203:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767955.142258:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767955/real 1775767955] req@ffff8f994f2e4e00 x1862022967373952/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767955.142283:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967373952/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767955.142294:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767955.142298:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767955.142302:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967373952:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767955.142330:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967374080:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775767955.142348:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767955.142367:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767955/real 1775767955] req@ffff8f994f2e7b80 x1862022967374080/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767955.142378:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967374080/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767955.142384:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767955.142387:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767955.142390:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967374080:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767955.142395:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967374208:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767955.142412:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767955.142419:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767955/real 1775767955] req@ffff8f994f2e4700 x1862022967374208/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767955.142428:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967374208/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767955.142433:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767955.142436:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767955.142438:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967374208:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767955.142442:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967374336:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767955.142453:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767955.142460:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767955/real 1775767955] req@ffff8f994f2e6300 x1862022967374336/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767955.142467:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967374336/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767955.142473:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767955.142475:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767955.142477:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967374336:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767956.037851:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767956.037926:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767957.061826:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767957.062006:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767958.085859:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767958.085870:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775767959.109828:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767959.109925:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767960.133861:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767960.133896:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767960.261894:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767960.261906:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767960.261913:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767960.261926:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4581 00000100:00080000:3.0:1775767960.261940:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767960.261948:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767960.261995:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967374464/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767960.262024:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767960.262027:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767960.262028:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767960.262032:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4581 00000100:00080000:3.0:1775767960.262035:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767960.262037:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767960.262048:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767960.262058:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967374592/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767960.262066:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767960.262071:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767960.262072:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775767960.262073:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967374464:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775767960.262077:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4581 00000100:00080000:3.0:1775767960.262081:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767960.262086:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767960.262089:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767960.262097:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967374720/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767960.262117:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767960.262120:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767960.262122:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767960.262125:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4581 00000100:00080000:3.0:1775767960.262127:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767960.262130:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767960.262133:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767960.262140:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967374848/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767960.262178:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767960.262233:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767960/real 1775767960] req@ffff8f9970edc000 x1862022967374464/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767960.262482:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967374464/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767960.262495:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767960.262500:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767960.262504:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967374464:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767960.262538:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967374592:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775767960.262627:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767960.262651:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767960/real 1775767960] req@ffff8f9970edc700 x1862022967374592/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767960.262664:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967374592/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767960.262673:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767960.262677:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767960.262680:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967374592:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767960.262687:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967374720:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767960.262710:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767960.262721:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767960/real 1775767960] req@ffff8f9970ede680 x1862022967374720/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767960.262732:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967374720/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767960.262796:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767960.262800:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767960.262804:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967374720:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767960.262811:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967374848:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767960.262835:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767960.262847:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767960/real 1775767960] req@ffff8f9970eded80 x1862022967374848/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767960.262859:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967374848/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767960.262868:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767960.262871:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767960.262875:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967374848:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767961.157841:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767961.157905:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767962.181859:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767962.181867:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775767963.205821:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767963.205922:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767964.229843:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767964.229910:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767965.253877:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767965.253884:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767965.381894:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767965.381905:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767965.381910:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767965.381921:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4586 00000100:00080000:2.0:1775767965.381933:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767965.381939:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767965.381978:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3800 x1862022967374976/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767965.382008:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767965.382012:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767965.382014:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767965.382018:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4586 00000100:00080000:2.0:1775767965.382023:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767965.382028:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767965.382038:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767965.382052:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0380 x1862022967375104/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767965.382068:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967374976:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775767965.382069:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767965.382079:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767965.382085:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767965.382091:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4586 00000100:00080000:2.0:1775767965.382094:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767965.382098:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767965.382104:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767965.382119:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0000 x1862022967375232/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767965.382130:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767965.382135:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767965.382136:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767965.382143:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4586 00000100:00080000:2.0:1775767965.382147:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767965.382151:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767965.382156:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767965.382170:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2680 x1862022967375360/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767965.382178:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767965.382256:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767965/real 1775767965] req@ffff8f99703d3800 x1862022967374976/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767965.382280:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3800 x1862022967374976/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767965.382294:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767965.382299:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767965.382477:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967374976:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767965.382505:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967375104:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775767965.382570:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767965.382589:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767965/real 1775767965] req@ffff8f99703d0380 x1862022967375104/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767965.382601:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0380 x1862022967375104/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767965.382609:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767965.382612:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767965.382614:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967375104:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767965.382618:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967375232:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767965.382631:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767965.382637:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767965/real 1775767965] req@ffff8f99703d0000 x1862022967375232/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767965.382644:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0000 x1862022967375232/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767965.382649:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767965.382651:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767965.382653:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967375232:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767965.382656:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967375360:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767965.382666:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767965.382692:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767965/real 1775767965] req@ffff8f99703d2680 x1862022967375360/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767965.382706:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2680 x1862022967375360/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767965.382714:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767965.382716:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767965.382719:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967375360:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767966.277880:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767966.277953:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767967.301825:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767967.301890:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767968.325881:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767968.325953:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767969.349812:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767969.349870:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767970.373823:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767970.373899:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767970.501834:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767970.501845:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767970.501851:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767970.501862:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4592 00000100:00080000:2.0:1775767970.501874:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767970.501881:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767970.501934:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2680 x1862022967375488/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767970.501964:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767970.501968:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767970.501971:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767970.501978:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4592 00000100:00080000:2.0:1775767970.501983:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767970.501987:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767970.501994:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967375488:192.168.202.146@tcp:250:kworker.0 00000008:00100000:2.0:1775767970.502002:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767970.502018:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0000 x1862022967375616/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767970.502027:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767970.502030:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767970.502032:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767970.502035:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4592 00000100:00080000:2.0:1775767970.502039:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767970.502042:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767970.502047:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767970.502055:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0380 x1862022967375744/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767970.502061:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767970.502064:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767970.502065:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767970.502067:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4592 00000100:00080000:2.0:1775767970.502069:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767970.502072:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767970.502074:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775767970.502081:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767970.502084:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3800 x1862022967375872/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767970.502147:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767970/real 1775767970] req@ffff8f99703d2680 x1862022967375488/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767970.502168:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2680 x1862022967375488/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767970.502179:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767970.502185:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767970.502190:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967375488:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767970.502218:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967375616:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775767970.502242:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767970.502257:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767970/real 1775767970] req@ffff8f99703d0000 x1862022967375616/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767970.502277:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0000 x1862022967375616/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767970.502415:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767970.502418:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767970.502420:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967375616:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767970.502425:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967375744:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767970.502443:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767970.502451:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767970/real 1775767970] req@ffff8f99703d0380 x1862022967375744/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767970.502458:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0380 x1862022967375744/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767970.502463:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767970.502465:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767970.502467:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967375744:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767970.502470:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967375872:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767970.502488:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767970.502515:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767970/real 1775767970] req@ffff8f99703d3800 x1862022967375872/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767970.502523:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3800 x1862022967375872/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767970.502528:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767970.502530:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767970.502533:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967375872:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767971.397813:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767971.397873:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767972.421911:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767972.421936:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767973.445853:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767973.445919:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767974.469887:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767974.469941:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767975.493860:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767975.493968:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767975.621912:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767975.621926:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767975.621932:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767975.621947:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4597 00000100:00080000:3.0:1775767975.621962:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767975.621971:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767975.622021:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967376000/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767975.622055:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767975.622067:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767975.622070:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767975.622076:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4597 00000100:00080000:3.0:1775767975.622082:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767975.622087:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767975.622102:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967376000:192.168.202.146@tcp:250:kworker.0 00000008:00100000:3.0:1775767975.622204:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767975.622225:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967376128/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767975.622246:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767975.622251:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767975.622254:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767975.622260:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4597 00000100:00080000:3.0:1775767975.622267:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775767975.622276:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967376128:192.168.202.146@tcp:38:kworker.0 00000100:00080000:3.0:1775767975.622277:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767975.622295:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767975.622320:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967376256/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767975.622340:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767975.622344:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767975.622346:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767975.622353:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4597 00000100:00080000:3.0:1775767975.622358:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775767975.622364:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967376256:192.168.202.146@tcp:8:kworker.0 00000100:00080000:3.0:1775767975.622366:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767975.622380:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767975.622403:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967376384/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767975.622438:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967376384:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767976.517891:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767976.517956:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767977.541859:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767977.541940:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767978.565819:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767978.565918:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775767978.693977:0:2366:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:1.0:1775767978.694014:0:2366:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775767978.694031:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775767978.694101:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775767978.694114:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775767978.694121:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:0.0:1775767978.694181:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767975/real 1775767978] req@ffff8f9970edd880 x1862022967376384/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767978.694208:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967376384/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767978.694222:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767978.694227:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767978.694232:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967376384:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767978.694240:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767975/real 1775767978] req@ffff8f9970edc380 x1862022967376256/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767978.694253:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967376256/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767978.694260:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767978.694263:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767978.694267:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967376256:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767978.694275:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767975/real 1775767978] req@ffff8f9970edf480 x1862022967376128/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767978.694287:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967376128/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767978.694294:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767978.694298:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767978.694306:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967376128:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767978.694313:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767975/real 1775767978] req@ffff8f9970edf800 x1862022967376000/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767978.694325:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967376000/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767978.694332:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767978.694335:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767978.694339:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967376000:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767979.589883:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767979.589905:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767980.613839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767980.613922:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767980.741868:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767980.741877:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767980.741882:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767980.741893:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4602 00000100:00080000:2.0:1775767980.741904:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767980.741912:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767980.741946:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0e00 x1862022967376512/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767980.741980:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767980.741985:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767980.741987:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767980.741992:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4602 00000100:00080000:2.0:1775767980.741997:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767980.742001:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767980.742012:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767980.742028:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0700 x1862022967376640/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767980.742039:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767980.742042:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767980.742044:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767980.742054:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4602 00000100:00080000:2.0:1775767980.742059:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767980.742063:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767980.742069:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767980.742083:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1500 x1862022967376768/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767980.742093:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767980.742098:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767980.742100:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767980.742104:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4602 00000100:00080000:2.0:1775767980.742108:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767980.742119:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767980.742123:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967376512:192.168.202.146@tcp:250:kworker.0 00000008:00100000:2.0:1775767980.742125:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767980.742142:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2d80 x1862022967376896/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767980.742199:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767980.742241:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767980/real 1775767980] req@ffff8f99703d0e00 x1862022967376512/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767980.742262:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0e00 x1862022967376512/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767980.742273:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767980.742278:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767980.742283:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967376512:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767980.742291:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967376640:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775767980.742316:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767980.742329:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767980/real 1775767980] req@ffff8f99703d0700 x1862022967376640/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767980.742340:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0700 x1862022967376640/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767980.742347:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767980.742350:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767980.742354:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967376640:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767980.742359:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967376768:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767980.742376:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767980.742385:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767980/real 1775767980] req@ffff8f99703d1500 x1862022967376768/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767980.742429:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1500 x1862022967376768/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767980.742438:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767980.742441:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767980.742444:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967376768:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767980.742478:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967376896:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767980.742501:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767980.742513:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767980/real 1775767980] req@ffff8f99703d2d80 x1862022967376896/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767980.742525:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2d80 x1862022967376896/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767980.742533:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767980.742537:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767980.742540:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967376896:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775767981.637874:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767981.637920:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775767982.661832:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767982.661952:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767983.685863:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767983.685951:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767984.709816:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767984.709917:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767985.733873:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767985.733938:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767985.861875:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767985.861888:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767985.861981:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767985.861993:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4607 00000100:00080000:3.0:1775767985.862007:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767985.862015:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767985.862064:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967377024/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767985.862110:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767985.862114:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767985.862116:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767985.862121:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4607 00000100:00080000:3.0:1775767985.862124:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767985.862129:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767985.862139:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767985.862153:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967377152/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767985.862163:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767985.862166:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767985.862168:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767985.862172:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4607 00000100:00080000:3.0:1775767985.862175:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767985.862179:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767985.862185:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767985.862186:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967377024:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775767985.862196:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967377280/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767985.862249:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767985.862252:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767985.862254:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767985.862258:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4607 00000800:00000100:0.0:1775767985.862262:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775767985.862265:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767985.862271:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767985.862277:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767985.862291:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967377408/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767985.862321:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767985/real 1775767985] req@ffff8f9970ede300 x1862022967377024/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767985.862339:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967377024/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767985.862369:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767985.862373:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767985.862376:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967377024:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767985.862382:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967377152:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775767985.862401:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767985.862411:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767985/real 1775767985] req@ffff8f9970edfb80 x1862022967377152/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767985.862463:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967377152/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767985.862469:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767985.862471:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767985.862473:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967377152:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767985.862498:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967377280:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767985.862513:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767985.862520:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767985/real 1775767985] req@ffff8f9970edca80 x1862022967377280/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767985.862527:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967377280/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767985.862532:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767985.862534:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767985.862536:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967377280:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767985.862539:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967377408:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767985.862549:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767985.862554:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767985/real 1775767985] req@ffff8f9970edea00 x1862022967377408/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767985.862560:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967377408/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767985.862597:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767985.862599:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767985.862601:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967377408:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767986.757878:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767986.757934:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767987.781842:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767987.781900:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767988.805813:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767988.805890:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767989.829849:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767989.829917:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767990.853806:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767990.853888:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767990.981896:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767990.981906:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767990.981911:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767990.981923:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4612 00000100:00080000:3.0:1775767990.981935:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767990.981942:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775767990.981989:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddf80 x1862022967377536/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767990.982016:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767990.982019:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767990.982020:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767990.982024:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4612 00000100:00080000:3.0:1775767990.982026:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767990.982029:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775767990.982058:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967377536:192.168.202.146@tcp:250:kworker.0 00000008:00100000:3.0:1775767990.982099:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767990.982113:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd180 x1862022967377664/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767990.982121:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767990.982123:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767990.982124:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767990.982127:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4612 00000100:00080000:3.0:1775767990.982130:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767990.982132:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767990.982136:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767990.982144:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967377792/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767990.982150:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767990.982152:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767990.982153:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767990.982155:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4612 00000100:00080000:3.0:1775767990.982158:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775767990.982163:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775767990.982163:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775767990.982167:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767990.982175:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967377920/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767990.982231:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767990/real 1775767990] req@ffff8f9970eddf80 x1862022967377536/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767990.982253:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddf80 x1862022967377536/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767990.982266:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767990.982271:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767990.982276:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967377536:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767990.982321:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967377664:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775767990.982348:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767990.982361:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767990/real 1775767990] req@ffff8f9970edd180 x1862022967377664/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767990.982374:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd180 x1862022967377664/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767990.982383:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767990.982387:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767990.982392:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967377664:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767990.982398:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967377792:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767990.982419:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767990.982429:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767990/real 1775767990] req@ffff8f9970edea00 x1862022967377792/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767990.982443:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967377792/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767990.982449:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767990.982451:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767990.982454:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967377792:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767990.982458:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967377920:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767990.982472:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767990.982479:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767990/real 1775767990] req@ffff8f9970edca80 x1862022967377920/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767990.982487:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967377920/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767990.982492:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767990.982494:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767990.982496:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967377920:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767991.877882:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767991.877976:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767992.901812:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767992.901917:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767993.925895:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767993.925924:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767994.949867:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767994.949947:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767995.973879:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767995.973957:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767996.101892:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767996.101907:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767996.101913:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767996.101935:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4617 00000100:00080000:2.0:1775767996.101952:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767996.101961:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775767996.102012:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3100 x1862022967378048/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767996.102037:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767996.102041:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767996.102043:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767996.102048:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4617 00000100:00080000:2.0:1775767996.102053:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767996.102057:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767996.102069:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767996.102076:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967378048:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775767996.102082:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2300 x1862022967378176/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767996.102093:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767996.102096:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767996.102098:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767996.102102:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4617 00000100:00080000:2.0:1775767996.102105:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767996.102109:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767996.102114:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767996.102124:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1180 x1862022967378304/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767996.102134:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767996.102137:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767996.102139:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767996.102144:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4617 00000100:00080000:2.0:1775767996.102146:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767996.102149:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775767996.102154:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775767996.102155:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775767996.102163:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3480 x1862022967378432/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767996.102207:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767996/real 1775767996] req@ffff8f99703d3100 x1862022967378048/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767996.102226:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3100 x1862022967378048/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767996.102235:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767996.102238:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767996.102242:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967378048:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775767996.102280:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967378176:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775767996.102300:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767996.102309:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767996/real 1775767996] req@ffff8f99703d2300 x1862022967378176/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767996.102318:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2300 x1862022967378176/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767996.102324:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767996.102326:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767996.102328:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967378176:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775767996.102332:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967378304:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767996.102344:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767996.102349:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767996/real 1775767996] req@ffff8f99703d1180 x1862022967378304/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767996.102362:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1180 x1862022967378304/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767996.102367:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767996.102369:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767996.102380:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967378304:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775767996.102389:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967378432:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775767996.102410:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775767996.102422:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767996/real 1775767996] req@ffff8f99703d3480 x1862022967378432/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767996.102434:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3480 x1862022967378432/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767996.102442:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767996.102445:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775767996.102449:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967378432:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775767996.997820:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767996.997888:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767998.021889:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775767998.021954:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767999.045895:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775767999.045960:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768000.069856:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768000.069965:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768001.093877:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768001.093887:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768001.222522:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768001.222535:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768001.222593:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768001.222605:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4622 00000100:00080000:3.0:1775768001.222617:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768001.222625:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768001.222678:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967378560/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768001.222718:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768001.222722:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768001.222724:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768001.222730:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4622 00000100:00080000:3.0:1775768001.222734:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768001.222767:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768001.222778:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768001.222785:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967378560:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775768001.222796:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967378688/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768001.222811:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768001.222816:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768001.222818:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768001.222822:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4622 00000100:00080000:3.0:1775768001.222828:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768001.222833:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768001.222838:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775768001.222854:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768001.222862:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967378816/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768001.222903:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768001/real 1775768001] req@ffff8f9970edfb80 x1862022967378560/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768001.222908:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768001.222919:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768001.222925:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768001.222932:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4622 00000100:00080000:3.0:1775768001.222937:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775768001.222939:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967378560/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768001.222944:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768001.222955:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1775768001.222958:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768001.222965:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768001.222973:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967378560:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775768001.222973:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967378944/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768001.223006:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967378688:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775768001.223031:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768001.223044:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768001/real 1775768001] req@ffff8f9970ede300 x1862022967378688/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768001.223058:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967378688/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768001.223066:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768001.223070:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768001.223074:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967378688:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775768001.223080:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967378816:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768001.223103:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768001.223114:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768001/real 1775768001] req@ffff8f9970edf800 x1862022967378816/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768001.223127:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967378816/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768001.223135:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768001.223139:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768001.223142:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967378816:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775768001.223149:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967378944:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768001.223168:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768001.223179:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768001/real 1775768001] req@ffff8f9970edf480 x1862022967378944/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768001.223192:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967378944/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768001.223221:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768001.223224:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768001.223228:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967378944:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768002.117839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768002.117924:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768003.141810:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768003.141887:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768004.165819:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768004.165875:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768005.189864:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768005.189932:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768006.213823:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768006.213895:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768006.341840:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768006.341849:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768006.341853:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768006.341862:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4627 00000100:00080000:3.0:1775768006.341872:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768006.341877:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768006.341908:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967379072/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768006.341931:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768006.341935:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768006.341937:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768006.341942:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4627 00000100:00080000:3.0:1775768006.341947:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768006.341952:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775768006.341963:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967379072:192.168.202.146@tcp:250:kworker.0 00000008:00100000:3.0:1775768006.342000:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768006.342014:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967379200/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768006.342030:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775768006.342031:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768006.342035:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768006.342036:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768006.342041:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4627 00000100:00080000:3.0:1775768006.342045:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768006.342049:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768006.342053:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768006.342062:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967379328/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768006.342068:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768006.342071:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768006.342072:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768006.342074:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4627 00000100:00080000:3.0:1775768006.342077:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768006.342081:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775768006.342082:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768006/real 1775768006] req@ffff8f9970edc380 x1862022967379072/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1775768006.342086:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768006.342098:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967379456/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768006.342105:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967379072/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768006.342119:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768006.342123:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768006.342126:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967379072:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775768006.342165:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967379200:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775768006.342193:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768006.342225:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768006/real 1775768006] req@ffff8f9970edd880 x1862022967379200/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768006.342234:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967379200/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768006.342240:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768006.342242:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768006.342245:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967379200:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775768006.342249:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967379328:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768006.342264:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768006.342271:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768006/real 1775768006] req@ffff8f9970eded80 x1862022967379328/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768006.342278:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967379328/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768006.342283:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768006.342285:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768006.342287:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967379328:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775768006.342290:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967379456:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768006.342300:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768006.342305:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768006/real 1775768006] req@ffff8f9970ede680 x1862022967379456/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768006.342312:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967379456/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768006.342318:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768006.342320:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768006.342322:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967379456:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768007.237890:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768007.237973:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768008.261866:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768008.261893:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768009.285840:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768009.285912:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768010.309807:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768010.309871:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768011.333833:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768011.333878:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768011.461815:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768011.461824:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768011.461859:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768011.461868:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4633 00000100:00080000:3.0:1775768011.461878:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768011.461884:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768011.461916:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967379584/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768011.461941:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768011.461944:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768011.461946:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768011.461950:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4633 00000100:00080000:3.0:1775768011.461953:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768011.461957:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768011.461965:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768011.461968:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967379584:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775768011.461987:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967379712/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768011.461997:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768011.462000:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768011.462002:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768011.462006:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4633 00000100:00080000:3.0:1775768011.462009:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768011.462013:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768011.462017:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768011.462028:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf100 x1862022967379840/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768011.462029:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775768011.462064:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768011.462067:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768011.462069:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768011.462073:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4633 00000100:00100000:0.0:1775768011.462076:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768011/real 1775768011] req@ffff8f9970edc700 x1862022967379584/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768011.462078:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768011.462085:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768011.462092:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768011.462115:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967379584/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768011.462116:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddc00 x1862022967379968/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768011.462144:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768011.462149:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768011.462153:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967379584:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775768011.462173:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967379712:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775768011.462189:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768011.462197:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768011/real 1775768011] req@ffff8f9970edc000 x1862022967379712/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768011.462205:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967379712/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768011.462209:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768011.462211:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768011.462227:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967379712:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775768011.462231:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967379840:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768011.462243:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768011.462249:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768011/real 1775768011] req@ffff8f9970edf100 x1862022967379840/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768011.462256:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf100 x1862022967379840/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768011.462260:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768011.462263:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768011.462266:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967379840:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775768011.462271:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967379968:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768011.462286:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768011.462296:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768011/real 1775768011] req@ffff8f9970eddc00 x1862022967379968/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768011.462307:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddc00 x1862022967379968/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768011.462314:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768011.462317:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768011.462319:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967379968:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775768012.357877:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768012.357908:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775768013.381815:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768013.381906:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768014.405874:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768014.405916:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768015.429889:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768015.430020:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768016.453829:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768016.453890:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768016.581839:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768016.581848:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768016.581852:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768016.581862:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4638 00000100:00080000:1.0:1775768016.581871:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768016.581877:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775768016.581909:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967380096/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768016.581939:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768016.581942:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768016.581944:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768016.581948:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4638 00000100:00080000:1.0:1775768016.581951:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768016.581955:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768016.581963:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768016.581976:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967380224/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775768016.581977:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967380096:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775768016.581992:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768016.581996:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768016.581997:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768016.582001:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4638 00000100:00080000:1.0:1775768016.582005:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768016.582009:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768016.582030:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775768016.582040:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768016.582044:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967380352/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768016.582064:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768016.582067:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768016.582069:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768016.582072:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4638 00000100:00080000:1.0:1775768016.582076:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768016.582079:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768016.582083:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768016.582091:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967380480/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775768016.582242:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768016/real 1775768016] req@ffff8f994f2e6300 x1862022967380096/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768016.582291:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967380096/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768016.582300:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768016.582303:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768016.582305:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967380096:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775768016.582326:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967380224:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775768016.582350:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768016.582380:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768016/real 1775768016] req@ffff8f994f2e4700 x1862022967380224/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768016.582391:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967380224/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768016.582399:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768016.582402:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768016.582405:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967380224:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775768016.582409:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967380352:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768016.582425:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768016.582432:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768016/real 1775768016] req@ffff8f994f2e7b80 x1862022967380352/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768016.582438:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967380352/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768016.582443:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768016.582445:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768016.582447:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967380352:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775768016.582450:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967380480:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768016.582459:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768016.582464:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768016/real 1775768016] req@ffff8f994f2e4e00 x1862022967380480/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768016.582470:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967380480/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768016.582489:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768016.582492:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768016.582495:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967380480:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768017.477855:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768017.477918:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768018.501817:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768018.501878:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768019.525836:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768019.525891:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768020.549817:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768020.549862:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768021.573821:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768021.573871:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768021.701817:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768021.701825:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768021.701829:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768021.701836:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4643 00000100:00080000:1.0:1775768021.701845:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768021.701850:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775768021.701881:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967380608/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768021.701902:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768021.701905:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768021.701906:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768021.701909:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4643 00000100:00080000:1.0:1775768021.701911:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768021.701913:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768021.701919:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768021.701927:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967380736/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768021.701933:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768021.701936:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768021.701937:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768021.701941:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4643 00000100:00100000:3.0:1775768021.701941:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967380608:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775768021.701944:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768021.701949:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768021.701952:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768021.701959:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967380864/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768021.701964:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768021.701966:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768021.701967:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768021.701970:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4643 00000100:00080000:1.0:1775768021.701972:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768021.701974:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:3.0:1775768021.702013:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:1.0:1775768021.702022:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768021.702032:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967380992/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775768021.702051:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768021/real 1775768021] req@ffff8f994f2e6680 x1862022967380608/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768021.702064:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967380608/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768021.702070:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768021.702073:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768021.702076:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967380608:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775768021.702092:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967380736:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775768021.702106:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768021.702113:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768021/real 1775768021] req@ffff8f994f2e4000 x1862022967380736/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768021.702119:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967380736/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768021.702125:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768021.702127:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768021.702128:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967380736:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775768021.702148:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967380864:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768021.702159:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768021.702165:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768021/real 1775768021] req@ffff8f994f2e5f80 x1862022967380864/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768021.702172:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967380864/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768021.702177:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768021.702178:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768021.702180:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967380864:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775768021.702183:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967380992:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768021.702192:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768021.702196:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768021/real 1775768021] req@ffff8f994f2e6a00 x1862022967380992/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768021.702202:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967380992/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768021.702206:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768021.702208:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768021.702209:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967380992:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768022.597874:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768022.597879:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768023.621830:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768023.621885:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768024.645817:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768024.645870:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768025.669797:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768025.669934:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768026.693848:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768026.693947:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768026.821887:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768026.821902:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768026.821907:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768026.821919:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4648 00000100:00080000:1.0:1775768026.821932:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768026.821939:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775768026.822015:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967381120/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768026.822049:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768026.822052:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768026.822053:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768026.822057:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4648 00000100:00080000:1.0:1775768026.822060:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768026.822063:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768026.822074:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768026.822087:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967381248/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768026.822098:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775768026.822127:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967381120:192.168.202.146@tcp:250:kworker.0 00000800:00000100:3.0:1775768026.822205:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768026.822262:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768026/real 1775768026] req@ffff8f994f2e6d80 x1862022967381120/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768026.822273:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768026.822278:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768026.822288:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4648 00000100:00080000:1.0:1775768026.822300:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775768026.822308:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967381120/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768026.822310:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00080000:3.0:1775768026.822323:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1775768026.822327:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1775768026.822328:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768026.822333:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967381120:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775768026.822340:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967381248:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775768026.822362:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967381376/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775768026.822362:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768026.822374:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768026/real 1775768026] req@ffff8f994f2e4a80 x1862022967381248/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768026.822385:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775768026.822387:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967381248/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768026.822396:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768026.822398:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768026.822406:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4648 00000100:00080000:3.0:1775768026.822409:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768026.822413:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768026.822414:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768026.822420:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967381248:192.168.202.146@tcp:38:kworker.0 00000100:00080000:1.0:1775768026.822425:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768026.822432:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768026.822447:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967381504/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775768026.822447:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967381376:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768026.822471:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768026.822480:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768026/real 1775768026] req@ffff8f994f2e7800 x1862022967381376/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768026.822488:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967381376/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768026.822510:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768026.822513:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768026.822515:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967381376:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775768026.822522:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967381504:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768026.822535:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768026.822541:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768026/real 1775768026] req@ffff8f994f2e5180 x1862022967381504/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768026.822549:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967381504/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768026.822554:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768026.822556:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768026.822558:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967381504:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768027.717822:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768027.717920:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768028.741860:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768028.741928:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768029.765877:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768029.765965:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768030.789887:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768030.789974:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768031.813839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768031.813878:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768031.941869:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768031.941878:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768031.941882:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768031.941891:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4653 00000100:00080000:3.0:1775768031.941914:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768031.941920:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768031.941954:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967381632/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768031.941978:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775768031.942009:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967381632:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775768031.942017:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768031.942019:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768031.942024:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4653 00000100:00080000:3.0:1775768031.942026:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768031.942029:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768031.942037:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768031.942048:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967381760/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768031.942056:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768031.942059:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768031.942060:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768031.942063:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4653 00000100:00080000:3.0:1775768031.942065:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768031.942069:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:0.0:1775768031.942070:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:3.0:1775768031.942075:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768031.942086:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddc00 x1862022967381888/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768031.942092:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768031.942095:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768031.942096:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768031.942099:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4653 00000100:00080000:3.0:1775768031.942101:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768031.942103:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768031.942116:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768031.942120:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768031/real 1775768031] req@ffff8f9970edce00 x1862022967381632/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768031.942134:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf100 x1862022967382016/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768031.942139:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967381632/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768031.942159:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768031.942163:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768031.942166:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967381632:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775768031.942187:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967381760:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775768031.942201:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768031.942209:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768031/real 1775768031] req@ffff8f9970edd500 x1862022967381760/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768031.942217:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967381760/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768031.942222:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768031.942224:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768031.942227:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967381760:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775768031.942230:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967381888:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768031.942242:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768031.942248:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768031/real 1775768031] req@ffff8f9970eddc00 x1862022967381888/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768031.942255:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddc00 x1862022967381888/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768031.942260:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768031.942262:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768031.942263:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967381888:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775768031.942267:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967382016:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768031.942276:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768031.942281:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768031/real 1775768031] req@ffff8f9970edf100 x1862022967382016/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768031.942288:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf100 x1862022967382016/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768031.942293:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768031.942294:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768031.942296:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967382016:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768032.837870:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768032.837946:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768033.861911:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768033.861965:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768034.885902:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768034.885958:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768035.909819:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768035.909931:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768036.933958:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768036.933994:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00080000:3.0:1775768037.061871:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768037.061883:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768037.061888:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768037.061899:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4658 00000100:00080000:3.0:1775768037.061912:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768037.061920:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768037.061965:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967382144/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768037.061994:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768037.061996:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768037.061998:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768037.062001:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4658 00000100:00080000:3.0:1775768037.062004:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768037.062006:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768037.062016:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768037.062028:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967382272/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768037.062036:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768037.062039:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768037.062040:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768037.062044:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4658 00000100:00100000:0.0:1775768037.062064:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967382144:192.168.202.146@tcp:250:kworker.0 00000800:00000100:0.0:1775768037.062152:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775768037.062161:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768037.062165:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768037.062172:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768037.062189:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967382400/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768037.062201:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768037.062206:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1775768037.062207:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768037/real 1775768037] req@ffff8f9970edc000 x1862022967382144/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768037.062209:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768037.062221:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4658 00000100:00080000:3.0:1775768037.062232:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768037.062239:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768037.062248:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768037.062256:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967382144/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768037.062269:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967382528/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768037.062273:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768037.062288:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768037.062298:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967382144:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775768037.062311:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967382272:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775768037.062338:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768037.062349:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768037/real 1775768037] req@ffff8f9970edc700 x1862022967382272/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768037.062357:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967382272/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768037.062363:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768037.062366:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768037.062368:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967382272:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775768037.062393:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967382400:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768037.062414:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768037.062434:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768037/real 1775768037] req@ffff8f9970ede680 x1862022967382400/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768037.062448:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967382400/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768037.062495:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768037.062498:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768037.062501:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967382400:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775768037.062506:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967382528:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768037.062523:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768037.062532:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768037/real 1775768037] req@ffff8f9970eded80 x1862022967382528/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768037.062539:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967382528/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768037.062544:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768037.062547:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768037.062549:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967382528:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768037.957911:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768037.957937:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768038.981901:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768038.981972:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768040.005868:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768040.005949:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768041.029879:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768041.030086:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768042.053821:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768042.053934:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768042.181871:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768042.181881:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768042.181886:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768042.181898:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4663 00000100:00080000:1.0:1775768042.181909:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768042.181915:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775768042.181951:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967382656/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768042.181976:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768042.181980:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768042.181983:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768042.181987:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4663 00000100:00080000:1.0:1775768042.181991:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768042.181996:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768042.182006:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768042.182016:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967382656:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775768042.182023:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967382784/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768042.182036:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768042.182039:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768042.182041:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768042.182044:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4663 00000100:00080000:1.0:1775768042.182048:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768042.182052:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768042.182057:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768042.182066:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967382912/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768042.182073:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768042.182076:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768042.182078:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768042.182082:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4663 00000100:00080000:1.0:1775768042.182086:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768042.182090:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768042.182095:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768042.182107:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967382784:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775768042.182109:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967383040/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768042.182136:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967382912:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775768042.182152:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967383040:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768043.077864:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768043.077964:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768044.101974:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768044.101975:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768045.125841:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768045.125945:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775768045.254132:0:2368:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:1.0:1775768045.254202:0:2368:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775768045.254222:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775768045.254264:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775768045.254273:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775768045.254281:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:0.0:1775768045.254315:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768042/real 1775768045] req@ffff8f994f2e6d80 x1862022967383040/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768045.254344:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967383040/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768045.254363:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768045.254369:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768045.254373:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967383040:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775768045.254382:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768042/real 1775768045] req@ffff8f994f2e7800 x1862022967382784/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768045.254397:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967382784/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768045.254406:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768045.254410:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768045.254413:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967382784:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775768045.254421:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768042/real 1775768045] req@ffff8f994f2e4a80 x1862022967382912/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768045.254433:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967382912/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768045.254442:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768045.254445:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768045.254449:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967382912:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775768045.254479:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768042/real 1775768045] req@ffff8f994f2e5180 x1862022967382656/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768045.254492:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967382656/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768045.254499:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768045.254502:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768045.254505:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967382656:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775768046.149873:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768046.149929:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768047.173843:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768047.173955:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768047.301908:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768047.301919:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768047.301924:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768047.301935:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4668 00000100:00080000:1.0:1775768047.301951:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768047.301961:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775768047.302024:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967383168/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768047.302058:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768047.302062:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768047.302065:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768047.302078:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4668 00000100:00080000:1.0:1775768047.302082:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768047.302086:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768047.302098:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768047.302115:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967383296/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768047.302136:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967383168:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775768047.302142:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768047.302151:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768047.302155:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768047.302161:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4668 00000100:00080000:1.0:1775768047.302167:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768047.302172:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768047.302179:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768047.302195:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967383424/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768047.302222:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775768047.302224:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768047.302230:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768047.302233:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768047.302240:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4668 00000100:00080000:1.0:1775768047.302248:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768047.302253:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768047.302260:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768047.302274:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967383552/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768047.302284:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768047/real 1775768047] req@ffff8f994f2e6a00 x1862022967383168/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768047.302312:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967383168/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768047.302325:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768047.302331:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768047.302335:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967383168:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775768047.302365:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967383296:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775768047.302408:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768047.302422:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768047/real 1775768047] req@ffff8f994f2e5f80 x1862022967383296/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768047.302441:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967383296/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768047.302495:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768047.302499:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768047.302503:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967383296:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775768047.302510:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967383424:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768047.302534:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768047.302547:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768047/real 1775768047] req@ffff8f994f2e4000 x1862022967383424/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768047.302560:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967383424/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768047.302569:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768047.302572:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768047.302576:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967383424:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775768047.302583:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967383552:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768047.302604:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768047.302616:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768047/real 1775768047] req@ffff8f994f2e6680 x1862022967383552/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768047.302628:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967383552/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768047.302637:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768047.302640:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768047.302644:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967383552:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768048.197899:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768048.197961:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768049.221825:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768049.221896:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768050.245851:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768050.245916:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768051.269844:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768051.269863:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768052.293820:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768052.294971:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775768052.421842:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768052.421852:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768052.421857:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768052.421867:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4674 00000100:00080000:2.0:1775768052.421879:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768052.421886:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775768052.421930:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3b80 x1862022967383680/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768052.421954:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768052.421958:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768052.421960:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768052.421964:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4674 00000100:00080000:2.0:1775768052.421968:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768052.421972:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768052.421981:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768052.421996:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1880 x1862022967383808/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768052.422007:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768052.422011:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775768052.422013:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768052.422027:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4674 00000100:00080000:2.0:1775768052.422031:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768052.422034:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768052.422041:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768052.422055:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2a00 x1862022967383936/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775768052.422056:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967383680:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775768052.422071:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768052.422075:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775768052.422077:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768052.422081:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4674 00000100:00080000:2.0:1775768052.422085:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768052.422089:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768052.422093:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768052.422104:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1c00 x1862022967384064/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775768052.422271:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768052.422340:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768052/real 1775768052] req@ffff8f99703d3b80 x1862022967383680/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768052.422357:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3b80 x1862022967383680/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768052.422366:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768052.422370:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768052.422374:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967383680:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775768052.422482:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967383808:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775768052.422537:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768052.422567:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768052/real 1775768052] req@ffff8f99703d1880 x1862022967383808/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768052.422586:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1880 x1862022967383808/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768052.422599:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768052.422605:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768052.422611:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967383808:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775768052.422643:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967383936:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768052.422662:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768052.422673:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768052/real 1775768052] req@ffff8f99703d2a00 x1862022967383936/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768052.422684:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2a00 x1862022967383936/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768052.422714:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768052.422719:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768052.422722:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967383936:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775768052.422728:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967384064:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768052.422795:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768052.422809:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768052/real 1775768052] req@ffff8f99703d1c00 x1862022967384064/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768052.422881:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1c00 x1862022967384064/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768052.422886:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768052.422888:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768052.422890:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967384064:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768053.317837:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768053.317896:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768054.341832:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768054.341862:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768055.365834:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768055.365881:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768056.389870:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768056.389946:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768057.413820:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768057.413861:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768057.541837:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768057.541844:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768057.541848:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768057.541861:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4679 00000100:00080000:3.0:1775768057.541869:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768057.541874:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768057.541900:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967384192/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768057.541921:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768057.541923:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768057.541924:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768057.541927:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4679 00000100:00080000:3.0:1775768057.541929:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768057.541931:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768057.541938:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768057.541945:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967384320/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768057.541969:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967384192:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775768057.541989:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768057.541991:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768057.541992:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768057.541995:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4679 00000100:00080000:3.0:1775768057.541997:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768057.542000:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768057.542003:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768057.542011:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967384448/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768057.542016:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768057.542019:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768057.542020:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768057.542021:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4679 00000100:00080000:3.0:1775768057.542023:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768057.542025:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768057.542027:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775768057.542034:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768057.542035:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967384576/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768057.542077:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768057/real 1775768057] req@ffff8f9970edd880 x1862022967384192/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768057.542089:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967384192/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768057.542097:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768057.542100:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768057.542103:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967384192:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775768057.542120:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967384320:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775768057.542134:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768057.542140:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768057/real 1775768057] req@ffff8f9970edc380 x1862022967384320/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768057.542148:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967384320/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768057.542153:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768057.542155:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768057.542157:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967384320:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775768057.542160:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967384448:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768057.542170:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768057.542174:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768057/real 1775768057] req@ffff8f9970edf480 x1862022967384448/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768057.542181:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967384448/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768057.542185:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768057.542187:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768057.542188:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967384448:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775768057.542191:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967384576:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768057.542199:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768057.542204:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768057/real 1775768057] req@ffff8f9970edf800 x1862022967384576/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768057.542210:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967384576/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768057.542214:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768057.542215:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768057.542217:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967384576:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768058.437890:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768058.437900:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768059.461816:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768059.461878:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768060.485844:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768060.485908:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768061.509822:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768061.509884:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768062.533839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768062.533898:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768062.661831:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768062.661841:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768062.661846:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768062.661856:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4684 00000100:00080000:3.0:1775768062.662018:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768062.662031:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768062.662069:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967384704/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768062.662096:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768062.662100:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768062.662103:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768062.662108:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4684 00000100:00080000:3.0:1775768062.662112:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768062.662115:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768062.662124:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768062.662126:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967384704:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775768062.662135:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967384832/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768062.662143:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768062.662145:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768062.662147:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768062.662149:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4684 00000100:00080000:3.0:1775768062.662151:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768062.662154:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768062.662158:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768062.662166:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967384960/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768062.662176:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1775768062.662184:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775768062.662200:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768062.662202:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768062.662207:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4684 00000100:00080000:3.0:1775768062.662211:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768062.662218:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768062.662224:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768062.662234:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768062/real 1775768062] req@ffff8f9970ede300 x1862022967384704/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768062.662236:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967385088/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768062.662271:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967384704/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768062.662281:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768062.662285:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768062.662290:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967384704:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775768062.662313:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967384832:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775768062.662334:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768062.662347:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768062/real 1775768062] req@ffff8f9970edfb80 x1862022967384832/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768062.662355:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967384832/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768062.662360:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768062.662362:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768062.662364:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967384832:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775768062.662367:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967384960:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768062.662379:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768062.662385:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768062/real 1775768062] req@ffff8f9970edca80 x1862022967384960/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768062.662411:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967384960/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768062.662416:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768062.662417:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768062.662419:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967384960:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775768062.662422:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967385088:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768062.662432:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768062.662438:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768062/real 1775768062] req@ffff8f9970edea00 x1862022967385088/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768062.662448:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967385088/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768062.662455:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768062.662458:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768062.662461:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967385088:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768063.557847:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768063.557920:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768064.581827:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768064.581905:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768065.605929:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768065.605930:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775768066.629818:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768066.629883:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768067.653837:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768067.653903:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768067.783181:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768067.783191:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768067.783197:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768067.783209:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4689 00000100:00080000:0.0:1775768067.783223:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768067.783243:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775768067.783318:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967385216/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768067.783351:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768067.783356:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768067.783358:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768067.783363:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4689 00000100:00080000:0.0:1775768067.783367:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768067.783371:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768067.783381:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768067.783396:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967385344/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768067.783405:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768067.783409:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768067.783410:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768067.783414:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4689 00000100:00080000:0.0:1775768067.783417:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768067.783421:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768067.783425:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768067.783433:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967385472/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768067.783442:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768067.783445:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775768067.783446:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768067.783467:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4689 00000100:00080000:0.0:1775768067.783471:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768067.783475:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768067.783480:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768067.783493:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967385600/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775768067.783568:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967385216:192.168.202.146@tcp:250:kworker.0 00000800:00000100:3.0:1775768067.783655:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768067.783700:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768067/real 1775768067] req@ffff8f9970a75180 x1862022967385216/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768067.783715:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967385216/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768067.783724:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768067.783727:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768067.783730:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967385216:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775768067.783736:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967385344:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775768067.783773:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768067.783782:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768067/real 1775768067] req@ffff8f9970a76300 x1862022967385344/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768067.783791:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967385344/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768067.783813:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768067.783815:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768067.783818:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967385344:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775768067.783823:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967385472:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768067.783845:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768067.783858:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768067/real 1775768067] req@ffff8f9970a75500 x1862022967385472/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768067.783870:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967385472/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768067.783879:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768067.783882:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768067.783884:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967385472:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775768067.783889:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967385600:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768067.783903:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768067.783911:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768067/real 1775768067] req@ffff8f9970a77480 x1862022967385600/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768067.783919:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967385600/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768067.783924:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768067.783926:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768067.783941:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967385600:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768068.677812:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768068.677856:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768069.701858:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768069.701935:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768070.725873:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768070.725955:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768071.749862:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768071.749950:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768072.773867:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768072.773879:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768072.901838:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768072.901849:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768072.901854:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768072.901865:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4694 00000100:00080000:0.0:1775768072.901876:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768072.901883:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775768072.901914:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967385728/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768072.901939:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768072.901942:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768072.901943:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768072.901947:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4694 00000100:00080000:0.0:1775768072.901949:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768072.901952:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768072.901959:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768072.901967:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967385856/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768072.901974:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768072.901977:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768072.901978:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768072.901980:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4694 00000100:00080000:0.0:1775768072.901982:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768072.901984:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768072.901986:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768072.901992:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967385984/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768072.901999:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768072.902001:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775768072.902002:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768072.902005:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4694 00000100:00080000:0.0:1775768072.902008:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768072.902010:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768072.902013:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768072.902019:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967385728:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775768072.902021:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967386112/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775768072.902081:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768072.902130:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768072/real 1775768072] req@ffff8f9970a75c00 x1862022967385728/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768072.902145:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967385728/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768072.902153:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768072.902157:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768072.902161:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967385728:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775768072.902168:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967385856:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775768072.902182:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768072.902189:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768072/real 1775768072] req@ffff8f9970a77100 x1862022967385856/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768072.902197:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967385856/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768072.902202:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768072.902204:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768072.902206:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967385856:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775768072.902210:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967385984:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768072.902220:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768072.902263:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768072/real 1775768072] req@ffff8f9970a76a00 x1862022967385984/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768072.902271:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967385984/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768072.902276:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768072.902278:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768072.902280:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967385984:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775768072.902300:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967386112:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768072.902312:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768072.902318:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768072/real 1775768072] req@ffff8f9970a74380 x1862022967386112/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768072.902325:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967386112/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768072.902330:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768072.902331:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768072.902333:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967386112:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768073.797819:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768073.797842:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768074.821831:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768074.821887:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768075.845837:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768075.845893:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768076.869820:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768076.869941:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768077.893886:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768077.893957:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768078.021879:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768078.021889:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768078.021895:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768078.021906:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4699 00000100:00080000:3.0:1775768078.021920:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768078.021927:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768078.022154:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd180 x1862022967386240/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768078.022193:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768078.022198:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768078.022200:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768078.022206:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4699 00000100:00080000:3.0:1775768078.022215:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768078.022220:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768078.022228:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768078.022239:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967386240:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775768078.022241:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddf80 x1862022967386368/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768078.022261:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768078.022264:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768078.022266:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768078.022270:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4699 00000100:00080000:3.0:1775768078.022274:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768078.022280:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768078.022286:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768078.022303:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967386496/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768078.022315:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768078.022319:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768078.022322:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1775768078.022336:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775768078.022352:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4699 00000100:00080000:3.0:1775768078.022357:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768078.022362:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768078.022368:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768078.022378:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967386624/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768078.022495:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768078/real 1775768078] req@ffff8f9970edd180 x1862022967386240/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768078.022511:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd180 x1862022967386240/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768078.022519:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768078.022522:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768078.022525:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967386240:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775768078.022545:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967386368:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775768078.022584:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768078.022592:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768078/real 1775768078] req@ffff8f9970eddf80 x1862022967386368/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768078.022609:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddf80 x1862022967386368/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768078.022615:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768078.022617:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768078.022619:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967386368:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775768078.022623:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967386496:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768078.022636:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768078.022642:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768078/real 1775768078] req@ffff8f9970edea00 x1862022967386496/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768078.022650:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967386496/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768078.022660:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768078.022662:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768078.022664:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967386496:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775768078.022674:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967386624:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768078.022688:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768078.022694:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768078/real 1775768078] req@ffff8f9970edca80 x1862022967386624/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768078.022702:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967386624/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768078.022707:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768078.022709:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768078.022711:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967386624:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768078.917881:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768078.917936:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768079.941920:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768079.941943:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775768080.965848:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768080.965892:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768081.989848:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768081.989894:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768083.013839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768083.013927:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768083.141883:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768083.141893:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768083.141898:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768083.141909:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4704 00000100:00080000:0.0:1775768083.141919:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768083.141925:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775768083.141956:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967386752/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768083.142014:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768083.142017:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768083.142018:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768083.142022:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4704 00000100:00080000:0.0:1775768083.142024:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768083.142027:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768083.142035:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768083.142044:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967386880/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768083.142052:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768083.142057:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768083.142058:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775768083.142058:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967386752:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775768083.142064:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4704 00000100:00080000:0.0:1775768083.142068:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768083.142072:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768083.142076:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768083.142097:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967387008/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768083.142108:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768083.142112:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775768083.142114:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768083.142118:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4704 00000100:00080000:0.0:1775768083.142124:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768083.142129:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768083.142134:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775768083.142142:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768083.142150:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967387136/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775768083.142196:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768083/real 1775768083] req@ffff8f9970a74700 x1862022967386752/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768083.142218:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967386752/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768083.142229:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768083.142234:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768083.142238:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967386752:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775768083.142270:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967386880:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775768083.142293:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768083.142305:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768083/real 1775768083] req@ffff8f9970a76d80 x1862022967386880/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768083.142317:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967386880/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768083.142325:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768083.142348:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768083.142360:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967386880:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775768083.142366:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967387008:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768083.142389:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768083.142401:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768083/real 1775768083] req@ffff8f9970a77800 x1862022967387008/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768083.142444:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967387008/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768083.142454:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768083.142457:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768083.142461:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967387008:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775768083.142467:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967387136:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768083.142490:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768083.142501:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768083/real 1775768083] req@ffff8f9970a74380 x1862022967387136/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768083.142527:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967387136/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768083.142536:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768083.142539:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768083.142543:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967387136:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768084.037813:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768084.037862:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768085.061841:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768085.061904:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768086.085818:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768086.085886:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768087.109906:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768087.109913:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768088.133810:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768088.133879:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768088.261918:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768088.261958:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768088.261964:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768088.261975:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4709 00000100:00080000:3.0:1775768088.261988:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768088.261994:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768088.262024:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967387264/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768088.262054:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768088.262057:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768088.262059:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768088.262064:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4709 00000100:00080000:3.0:1775768088.262067:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768088.262070:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768088.262077:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768088.262088:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967387392/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768088.262095:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768088.262098:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768088.262099:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768088.262102:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4709 00000100:00080000:3.0:1775768088.262105:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775768088.262108:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967387264:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775768088.262110:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768088.262117:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768088.262125:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967387520/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768088.262177:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775768088.262204:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768088.262207:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768088.262208:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768088.262213:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4709 00000100:00100000:0.0:1775768088.262217:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768088/real 1775768088] req@ffff8f9970edfb80 x1862022967387264/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768088.262218:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768088.262225:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768088.262232:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768088.262249:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967387264/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768088.262257:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768088.262261:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768088.262264:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967387264:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775768088.262270:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967387392:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775768088.262289:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768088.262300:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768088/real 1775768088] req@ffff8f9970ede300 x1862022967387392/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768088.262309:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967387392/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768088.262320:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768088.262323:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768088.262325:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967387392:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775768088.262347:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967387648/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768088.262349:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967387520:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768088.262394:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768088.262421:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768088/real 1775768088] req@ffff8f9970edf800 x1862022967387520/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768088.262440:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967387520/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768088.262450:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768088.262454:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768088.262459:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967387520:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775768088.262476:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967387648:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768088.262499:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768088.262511:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768088/real 1775768088] req@ffff8f9970edf480 x1862022967387648/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768088.262524:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967387648/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768088.262533:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768088.262536:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768088.262541:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967387648:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768089.157880:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768089.157933:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768090.181810:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768090.181889:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768091.205844:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768091.205907:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768092.229845:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768092.229902:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768093.253858:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768093.253929:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768093.381886:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768093.381894:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768093.381898:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768093.381907:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4714 00000100:00080000:0.0:1775768093.381917:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768093.381922:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775768093.381954:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967387776/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768093.381988:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768093.381992:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768093.381994:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768093.381998:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4714 00000100:00080000:0.0:1775768093.382001:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768093.382004:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768093.382012:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768093.382016:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967387776:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775768093.382024:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967387904/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768093.382033:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768093.382036:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768093.382037:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768093.382040:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4714 00000100:00080000:0.0:1775768093.382044:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768093.382047:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768093.382061:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768093.382069:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967388032/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768093.382076:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768093.382079:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775768093.382080:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768093.382084:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4714 00000800:00000100:3.0:1775768093.382088:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775768093.382090:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768093.382094:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768093.382098:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768093.382106:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967388160/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775768093.382132:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768093/real 1775768093] req@ffff8f9970a76a00 x1862022967387776/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768093.382157:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967387776/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768093.382171:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768093.382175:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768093.382178:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967387776:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775768093.382196:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967387904:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775768093.382223:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768093.382235:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768093/real 1775768093] req@ffff8f9970a77100 x1862022967387904/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768093.382302:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967387904/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768093.382323:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768093.382325:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768093.382328:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967387904:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775768093.382333:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967388032:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768093.382351:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768093.382359:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768093/real 1775768093] req@ffff8f9970a75c00 x1862022967388032/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768093.382367:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967388032/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768093.382372:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768093.382374:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768093.382376:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967388032:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775768093.382380:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967388160:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768093.382391:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768093.382397:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768093/real 1775768093] req@ffff8f9970a77480 x1862022967388160/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768093.382404:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967388160/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768093.382426:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768093.382428:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768093.382430:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967388160:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768094.277905:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768094.277908:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768095.301843:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768095.301943:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768096.325822:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768096.325888:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768097.349838:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768097.349888:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768098.373834:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768098.373898:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775768098.501859:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768098.501869:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768098.501875:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768098.501884:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4720 00000100:00080000:2.0:1775768098.501894:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768098.501899:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775768098.501939:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1c00 x1862022967388288/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768098.501972:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768098.501975:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768098.501977:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768098.501981:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4720 00000100:00080000:2.0:1775768098.501984:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768098.501987:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768098.501996:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768098.502007:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2a00 x1862022967388416/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768098.502015:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768098.502018:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775768098.502020:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775768098.502023:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967388288:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775768098.502025:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4720 00000100:00080000:2.0:1775768098.502031:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768098.502034:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768098.502038:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768098.502047:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1880 x1862022967388544/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768098.502058:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768098.502061:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775768098.502062:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768098.502064:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4720 00000100:00080000:2.0:1775768098.502070:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768098.502073:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768098.502075:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775768098.502083:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768098.502084:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3b80 x1862022967388672/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775768098.502131:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768098/real 1775768098] req@ffff8f99703d1c00 x1862022967388288/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768098.502147:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1c00 x1862022967388288/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768098.502156:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768098.502164:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768098.502166:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967388288:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775768098.502185:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967388416:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775768098.502201:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768098.502209:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768098/real 1775768098] req@ffff8f99703d2a00 x1862022967388416/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768098.502217:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2a00 x1862022967388416/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768098.502223:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768098.502225:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768098.502227:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967388416:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775768098.502249:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967388544:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768098.502282:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768098.502291:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768098/real 1775768098] req@ffff8f99703d1880 x1862022967388544/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768098.502299:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1880 x1862022967388544/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768098.502305:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768098.502307:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768098.502309:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967388544:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775768098.502316:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967388672:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768098.502334:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768098.502344:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768098/real 1775768098] req@ffff8f99703d3b80 x1862022967388672/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768098.502356:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3b80 x1862022967388672/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768098.502365:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768098.502369:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768098.502373:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967388672:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768099.397856:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768099.397918:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768100.421851:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768100.421896:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768101.445875:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768101.445889:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768102.469811:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768102.469862:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768103.493800:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768103.493858:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768103.621827:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768103.621835:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768103.621838:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768103.621882:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4725 00000100:00080000:3.0:1775768103.621891:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768103.621896:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768103.621925:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967388800/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768103.621945:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768103.621947:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768103.621948:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768103.621952:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4725 00000100:00080000:3.0:1775768103.621953:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768103.621956:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768103.621963:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768103.621971:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967388928/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768103.621977:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768103.621979:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768103.621981:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768103.621984:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4725 00000100:00100000:0.0:1775768103.621985:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967388800:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775768103.621986:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768103.621990:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768103.621996:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768103.622001:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967389056/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768103.622019:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768103.622022:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768103.622023:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768103.622025:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4725 00000100:00080000:3.0:1775768103.622027:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768103.622029:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768103.622031:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768103.622038:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967389184/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768103.622055:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768103.622094:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768103/real 1775768103] req@ffff8f9970edc380 x1862022967388800/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768103.622107:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967388800/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768103.622119:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768103.622122:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768103.622125:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967388800:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775768103.622142:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967388928:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775768103.622157:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768103.622164:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768103/real 1775768103] req@ffff8f9970edd880 x1862022967388928/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768103.622171:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967388928/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768103.622176:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768103.622178:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768103.622180:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967388928:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775768103.622183:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967389056:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768103.622193:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768103.622198:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768103/real 1775768103] req@ffff8f9970eded80 x1862022967389056/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768103.622233:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967389056/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768103.622242:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768103.622245:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768103.622248:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967389056:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775768103.622272:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967389184:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768103.622289:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768103.622298:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768103/real 1775768103] req@ffff8f9970ede680 x1862022967389184/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768103.622305:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967389184/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768103.622317:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768103.622319:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768103.622323:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967389184:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768104.517850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768104.517897:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768105.541819:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768105.541901:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768106.565869:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768106.565937:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768107.589811:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768107.589865:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768108.613908:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768108.613923:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768108.741848:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768108.741859:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768108.741864:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768108.741874:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4730 00000100:00080000:3.0:1775768108.741886:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768108.741893:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768108.741931:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967389312/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768108.741962:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768108.741967:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768108.741969:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768108.741974:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4730 00000100:00080000:3.0:1775768108.741979:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768108.741983:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775768108.742002:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967389312:192.168.202.146@tcp:250:kworker.0 00000008:00100000:3.0:1775768108.742030:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768108.742047:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967389440/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768108.742061:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768108.742065:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768108.742067:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768108.742071:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4730 00000100:00080000:3.0:1775768108.742077:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768108.742082:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768108.742088:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768108.742103:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf100 x1862022967389568/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768108.742116:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768108.742121:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1775768108.742123:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967389440:192.168.202.146@tcp:38:kworker.0 00000100:00080000:3.0:1775768108.742124:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768108.742135:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4730 00000100:00080000:3.0:1775768108.742141:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768108.742146:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768108.742153:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768108.742180:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967389568:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775768108.742187:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddc00 x1862022967389696/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768108.742213:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967389696:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768109.637824:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768109.637862:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768110.661880:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768110.661967:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768111.685867:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768111.685956:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775768111.814110:0:2367:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:1.0:1775768111.814183:0:2367:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775768111.814204:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775768111.814256:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775768111.814267:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775768111.814275:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:0.0:1775768111.814331:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768108/real 1775768111] req@ffff8f9970eddc00 x1862022967389696/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768111.814366:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddc00 x1862022967389696/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768111.814388:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768111.814395:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768111.814401:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967389696:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775768111.814412:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768108/real 1775768111] req@ffff8f9970edc000 x1862022967389440/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768111.814427:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967389440/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768111.814437:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768111.814442:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768111.814446:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967389440:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775768111.814454:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768108/real 1775768111] req@ffff8f9970edf100 x1862022967389568/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768111.814467:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf100 x1862022967389568/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768111.814475:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768111.814479:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768111.814482:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967389568:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775768111.814490:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768108/real 1775768111] req@ffff8f9970edc700 x1862022967389312/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768111.814504:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967389312/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768111.814954:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768111.814964:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768111.814972:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967389312:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775768112.709842:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768112.709938:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768113.733846:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768113.734017:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768113.861863:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768113.861874:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768113.861921:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768113.861932:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4735 00000100:00080000:3.0:1775768113.861943:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768113.861951:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768113.861989:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967389824/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768113.862023:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768113.862027:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768113.862030:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768113.862035:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4735 00000100:00080000:3.0:1775768113.862039:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768113.862045:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768113.862055:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768113.862072:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967389952/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768113.862075:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967389824:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775768113.862093:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768113.862097:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768113.862099:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768113.862104:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4735 00000100:00080000:3.0:1775768113.862127:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768113.862134:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768113.862141:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768113.862157:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967390080/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768113.862164:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775768113.862200:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768113.862204:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768113.862206:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768113.862212:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4735 00000100:00100000:0.0:1775768113.862218:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768113/real 1775768113] req@ffff8f9970edd500 x1862022967389824/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768113.862218:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768113.862226:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768113.862238:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768113.862249:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967389824/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768113.862255:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf100 x1862022967390208/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768113.862265:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768113.862270:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768113.862284:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967389824:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775768113.862315:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967389952:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775768113.862342:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768113.862356:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768113/real 1775768113] req@ffff8f9970edce00 x1862022967389952/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768113.862370:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967389952/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768113.862378:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768113.862381:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768113.862385:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967389952:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775768113.862392:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967390080:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768113.862413:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768113.862423:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768113/real 1775768113] req@ffff8f9970edc700 x1862022967390080/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768113.862435:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967390080/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768113.862444:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768113.862448:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768113.862451:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967390080:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775768113.862458:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967390208:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768113.862479:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768113.862490:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768113/real 1775768113] req@ffff8f9970edf100 x1862022967390208/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768113.862501:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf100 x1862022967390208/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768113.862510:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768113.862514:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768113.862517:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967390208:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768114.757858:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768114.757948:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768115.781868:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768115.781878:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775768116.805849:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768116.805888:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768117.829848:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768117.829903:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768118.853917:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768118.853917:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00080000:0.0:1775768118.981846:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768118.981854:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768118.981857:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768118.981865:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4740 00000100:00080000:0.0:1775768118.981874:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768118.981879:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775768118.981910:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967390336/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768118.981936:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768118.981939:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768118.981941:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768118.981945:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4740 00000100:00080000:0.0:1775768118.981949:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768118.981954:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768118.981962:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768118.981967:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967390336:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775768118.981976:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967390464/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768118.981987:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768118.981991:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768118.981992:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768118.981996:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4740 00000100:00080000:0.0:1775768118.982000:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768118.982004:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768118.982009:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768118.982021:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967390592/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775768118.982025:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775768118.982076:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768118.982078:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775768118.982080:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768118.982083:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4740 00000100:00080000:0.0:1775768118.982086:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768118.982090:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768118.982092:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768118/real 1775768118] req@ffff8f9970a75500 x1862022967390336/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1775768118.982095:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768118.982110:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967390720/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775768118.982117:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967390336/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768118.982129:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768118.982134:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768118.982138:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967390336:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775768118.982159:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967390464:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775768118.982178:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768118.982189:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768118/real 1775768118] req@ffff8f9970a76300 x1862022967390464/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768118.982217:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967390464/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768118.982225:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768118.982228:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768118.982231:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967390464:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775768118.982236:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967390592:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768118.982253:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768118.982261:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768118/real 1775768118] req@ffff8f9970a75180 x1862022967390592/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768118.982271:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967390592/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768118.982278:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768118.982281:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768118.982285:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967390592:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775768118.982289:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967390720:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768118.982303:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768118.982311:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768118/real 1775768118] req@ffff8f9970a77480 x1862022967390720/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768118.982321:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967390720/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768118.982346:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768118.982349:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768118.982352:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967390720:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768119.877828:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768119.877887:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768120.901842:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768120.901886:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768121.925804:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768121.925994:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768122.949890:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768122.949919:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768123.973883:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768123.973974:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768124.101868:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768124.101876:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768124.101879:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768124.101889:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4745 00000100:00080000:0.0:1775768124.101898:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768124.101903:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775768124.101932:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967390848/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768124.101954:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768124.101957:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768124.101958:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768124.101962:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4745 00000100:00100000:2.0:1775768124.101983:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967390848:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775768124.101993:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768124.101997:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768124.102005:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768124.102030:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967390976/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768124.102038:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768124.102040:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768124.102042:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768124.102045:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4745 00000100:00080000:0.0:1775768124.102049:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768124.102051:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768124.102055:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775768124.102055:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768124.102066:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967391104/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768124.102073:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768124.102075:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775768124.102076:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768124.102080:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4745 00000100:00080000:0.0:1775768124.102081:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768124.102084:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768124.102087:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768124.102107:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967391232/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768124.102109:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768124/real 1775768124] req@ffff8f9970a75c00 x1862022967390848/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768124.102140:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967390848/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768124.102151:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768124.102158:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768124.102166:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967390848:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775768124.102199:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967390976:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775768124.102226:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768124.102238:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768124/real 1775768124] req@ffff8f9970a77100 x1862022967390976/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768124.102283:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967390976/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768124.102295:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768124.102299:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768124.102303:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967390976:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775768124.102312:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967391104:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775768124.102337:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768124.102350:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768124/real 1775768124] req@ffff8f9970a76a00 x1862022967391104/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768124.102370:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967391104/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768124.102376:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768124.102378:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768124.102381:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967391104:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768124.102390:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967391232:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775768124.102404:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768124.102414:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768124/real 1775768124] req@ffff8f9970a74380 x1862022967391232/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768124.102431:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967391232/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768124.102441:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768124.102447:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768124.102458:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967391232:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768124.997849:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768124.997938:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768126.021833:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768126.021893:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768127.045837:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768127.045901:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768128.069843:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768128.069893:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768129.093808:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768129.093850:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768129.221856:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768129.221863:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768129.221867:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768129.221875:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4750 00000100:00080000:3.0:1775768129.221884:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768129.221888:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768129.221919:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967391360/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768129.221943:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768129.221947:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768129.221948:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768129.221953:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4750 00000100:00080000:3.0:1775768129.221976:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768129.221980:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775768129.221986:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967391360:192.168.202.146@tcp:250:kworker.0 00000008:00100000:3.0:1775768129.221988:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768129.222000:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddc00 x1862022967391488/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768129.222007:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768129.222010:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768129.222029:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768129.222032:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4750 00000100:00080000:3.0:1775768129.222036:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775768129.222039:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775768129.222041:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768129.222046:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768129.222055:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967391616/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768129.222063:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768129.222067:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768129.222068:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768129.222072:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4750 00000100:00080000:3.0:1775768129.222075:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768129.222078:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775768129.222081:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768129/real 1775768129] req@ffff8f9970edc000 x1862022967391360/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1775768129.222086:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768129.222098:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967391360/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768129.222102:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967391744/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768129.222113:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768129.222118:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768129.222121:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967391360:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775768129.222139:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967391488:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775768129.222152:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768129.222159:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768129/real 1775768129] req@ffff8f9970eddc00 x1862022967391488/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768129.222166:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddc00 x1862022967391488/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768129.222171:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768129.222173:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768129.222175:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967391488:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775768129.222178:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967391616:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775768129.222189:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768129.222193:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768129/real 1775768129] req@ffff8f9970ede680 x1862022967391616/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768129.222199:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967391616/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768129.222204:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768129.222205:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768129.222207:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967391616:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768129.222210:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967391744:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775768129.222218:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768129.222222:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768129/real 1775768129] req@ffff8f9970eded80 x1862022967391744/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768129.222228:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967391744/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768129.222233:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768129.222234:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768129.222236:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967391744:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768130.117845:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768130.117862:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768131.141833:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768131.141879:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768132.165837:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768132.165888:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768133.189856:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768133.189938:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768134.213880:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768134.213954:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768134.342081:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768134.342095:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768134.342110:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768134.342124:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4755 00000100:00080000:3.0:1775768134.342139:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768134.342147:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768134.342191:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967391872/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768134.342484:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768134.342490:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768134.342493:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768134.342503:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4755 00000100:00080000:3.0:1775768134.342515:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768134.342521:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768134.342536:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768134.342570:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967392000/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768134.342581:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768134.342584:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768134.342585:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768134.342589:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4755 00000100:00080000:3.0:1775768134.342592:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768134.342596:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768134.342600:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768134.342624:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967392128/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768134.342633:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768134.342636:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768134.342638:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768134.342641:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4755 00000100:00080000:3.0:1775768134.342664:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768134.342669:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768134.342673:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768134.342683:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967392256/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768134.342820:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967391872:192.168.202.146@tcp:250:kworker.0 00000800:00000100:1.0:1775768134.342900:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768134.342973:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768134/real 1775768134] req@ffff8f9970edd880 x1862022967391872/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768134.342998:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967391872/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768134.343009:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768134.343014:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768134.343018:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967391872:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775768134.343025:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967392000:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775768134.343046:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768134.343057:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768134/real 1775768134] req@ffff8f9970edc380 x1862022967392000/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768134.343067:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967392000/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768134.343075:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768134.343077:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768134.343080:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967392000:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775768134.343087:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967392128:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775768134.343104:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768134.343114:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768134/real 1775768134] req@ffff8f9970edf480 x1862022967392128/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768134.343124:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967392128/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768134.343131:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768134.343134:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768134.343137:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967392128:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775768134.343142:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967392256:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775768134.343159:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768134.343167:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768134/real 1775768134] req@ffff8f9970edf800 x1862022967392256/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768134.343177:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967392256/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768134.343183:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768134.343186:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768134.343188:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967392256:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768135.237864:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768135.237918:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768136.261925:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768136.262002:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768137.285937:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768137.285944:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768138.309863:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768138.309966:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768139.333852:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768139.333903:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768139.461892:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768139.462015:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768139.462019:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768139.462026:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4761 00000100:00080000:1.0:1775768139.462036:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768139.462041:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775768139.462076:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967392384/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768139.462108:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768139.462111:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768139.462113:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768139.462117:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4761 00000100:00080000:1.0:1775768139.462121:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768139.462125:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768139.462133:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768139.462145:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967392512/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768139.462149:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967392384:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775768139.462163:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768139.462166:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768139.462167:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768139.462171:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4761 00000100:00080000:1.0:1775768139.462174:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768139.462178:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768139.462182:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768139.462200:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967392640/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775768139.462203:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775768139.462210:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768139.462213:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768139.462215:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768139.462218:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4761 00000100:00080000:1.0:1775768139.462221:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768139.462223:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768139.462227:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768139.462234:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967392768/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768139.462238:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768139/real 1775768139] req@ffff8f994f2e4e00 x1862022967392384/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768139.462254:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967392384/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768139.462262:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768139.462265:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768139.462268:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967392384:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775768139.462283:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967392512:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775768139.462296:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768139.462309:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768139/real 1775768139] req@ffff8f994f2e7b80 x1862022967392512/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768139.462316:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967392512/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768139.462321:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768139.462323:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768139.462325:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967392512:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775768139.462328:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967392640:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775768139.462339:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768139.462344:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768139/real 1775768139] req@ffff8f994f2e4700 x1862022967392640/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768139.462350:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967392640/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768139.462354:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768139.462356:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768139.462357:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967392640:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768139.462360:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967392768:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775768139.462368:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768139.462373:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768139/real 1775768139] req@ffff8f994f2e6300 x1862022967392768/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768139.462379:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967392768/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768139.462383:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768139.462385:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768139.462386:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967392768:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768140.357878:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768140.358032:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768141.381855:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768141.381912:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768142.405915:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768142.405964:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775768143.429814:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768143.429914:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768144.453870:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768144.453878:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00080000:1.0:1775768144.581797:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768144.581806:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768144.581821:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768144.581831:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4766 00000100:00080000:1.0:1775768144.581841:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768144.581847:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775768144.581882:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967392896/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768144.581912:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768144.581916:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768144.581917:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768144.581921:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4766 00000100:00080000:1.0:1775768144.581925:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768144.581928:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768144.581938:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768144.581949:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967393024/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768144.581959:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967392896:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775768144.581962:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768144.581966:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768144.581968:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768144.581973:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4766 00000100:00080000:1.0:1775768144.581977:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768144.581981:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768144.581986:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768144.581996:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967393152/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768144.582005:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768144.582010:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:2.0:1775768144.582010:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775768144.582014:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768144.582018:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4766 00000100:00080000:1.0:1775768144.582022:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768144.582026:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768144.582031:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768144.582040:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967393280/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768144.582053:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768144/real 1775768144] req@ffff8f994f2e6300 x1862022967392896/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768144.582075:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967392896/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768144.582085:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768144.582089:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768144.582092:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967392896:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775768144.582109:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967393024:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775768144.582124:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768144.582132:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768144/real 1775768144] req@ffff8f994f2e4700 x1862022967393024/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768144.582139:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967393024/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768144.582143:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768144.582145:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768144.582147:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967393024:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775768144.582150:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967393152:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775768144.582160:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768144.582164:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768144/real 1775768144] req@ffff8f994f2e7b80 x1862022967393152/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768144.582175:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967393152/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768144.582180:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768144.582181:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768144.582183:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967393152:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768144.582186:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967393280:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775768144.582193:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768144.582198:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768144/real 1775768144] req@ffff8f994f2e4e00 x1862022967393280/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768144.582218:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967393280/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768144.582226:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768144.582229:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768144.582232:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967393280:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768145.477861:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768145.477999:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768146.501852:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768146.501917:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768147.525883:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768147.525927:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768148.549809:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768148.549899:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768149.573847:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768149.573922:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768149.701855:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768149.701861:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768149.701864:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768149.701871:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4771 00000100:00080000:3.0:1775768149.701885:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768149.701890:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768149.701917:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967393408/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768149.701941:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768149.701945:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768149.701946:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768149.701951:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4771 00000100:00080000:3.0:1775768149.701954:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768149.701958:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768149.701966:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768149.701979:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967393536/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768149.701985:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967393408:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775768149.701990:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768149.702013:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768149.702014:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768149.702018:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4771 00000100:00080000:3.0:1775768149.702028:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768149.702032:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768149.702036:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775768149.702040:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768149.702046:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967393664/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768149.702053:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768149.702055:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768149.702056:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768149.702058:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4771 00000100:00080000:3.0:1775768149.702061:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768149.702063:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768149.702065:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768149.702071:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967393792/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768149.702079:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768149/real 1775768149] req@ffff8f9970ede300 x1862022967393408/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768149.702098:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967393408/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768149.702105:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768149.702108:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768149.702111:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967393408:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775768149.702127:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967393536:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775768149.702139:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768149.702146:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768149/real 1775768149] req@ffff8f9970edfb80 x1862022967393536/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768149.702153:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967393536/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768149.702158:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768149.702160:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768149.702161:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967393536:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775768149.702164:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967393664:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775768149.702173:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768149.702178:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768149/real 1775768149] req@ffff8f9970edca80 x1862022967393664/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768149.702184:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967393664/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768149.702196:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768149.702200:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768149.702202:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967393664:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768149.702207:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967393792:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775768149.702224:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768149.702233:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768149/real 1775768149] req@ffff8f9970edea00 x1862022967393792/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768149.702244:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967393792/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768149.702249:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768149.702251:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768149.702252:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967393792:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768150.597882:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768150.598050:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768151.621893:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768151.621924:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768152.645839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768152.645916:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768153.669835:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768153.669916:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768154.693837:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768154.693899:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768154.821818:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768154.821833:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768154.821838:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768154.821849:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4776 00000100:00080000:3.0:1775768154.821862:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768154.821868:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768154.821908:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddf80 x1862022967393920/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768154.822103:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768154.822108:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768154.822111:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768154.822119:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4776 00000100:00080000:3.0:1775768154.822130:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768154.822137:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768154.822149:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768154.822150:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967393920:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775768154.822190:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd180 x1862022967394048/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768154.822202:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768154.822205:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768154.822206:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768154.822212:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4776 00000100:00080000:3.0:1775768154.822216:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768154.822222:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768154.822228:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775768154.822229:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768154.822247:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967394176/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768154.822257:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768154.822260:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768154.822262:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768154.822267:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4776 00000100:00080000:3.0:1775768154.822289:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775768154.822290:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768154/real 1775768154] req@ffff8f9970eddf80 x1862022967393920/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768154.822302:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768154.822318:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768154.822334:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967394304/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768154.822335:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddf80 x1862022967393920/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768154.822385:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768154.822400:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768154.822406:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967393920:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775768154.822436:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967394048:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775768154.822456:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768154.822465:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768154/real 1775768154] req@ffff8f9970edd180 x1862022967394048/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768154.822473:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd180 x1862022967394048/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768154.822478:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768154.822481:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768154.822483:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967394048:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775768154.822487:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967394176:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775768154.822513:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768154.822522:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768154/real 1775768154] req@ffff8f9970edea00 x1862022967394176/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768154.822534:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967394176/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768154.822542:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768154.822545:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768154.822548:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967394176:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768154.822555:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967394304:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775768154.822574:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768154.822592:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768154/real 1775768154] req@ffff8f9970edca80 x1862022967394304/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768154.822604:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967394304/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768154.822610:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768154.822613:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768154.822615:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967394304:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768155.717844:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768155.717910:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768156.741835:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768156.741881:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768157.765855:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768157.765917:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768158.789906:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775768158.790073:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:3.0:1775768159.813848:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768159.813949:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768159.941857:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768159.941867:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768159.941872:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768159.941884:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4781 00000100:00080000:1.0:1775768159.941897:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768159.941904:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775768159.941944:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967394432/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768159.941974:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768159.941976:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768159.941978:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768159.941981:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4781 00000100:00080000:1.0:1775768159.941984:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768159.941986:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768159.941997:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768159.942015:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967394432:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775768159.942018:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967394560/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768159.942040:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768159.942044:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768159.942046:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768159.942049:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4781 00000100:00080000:1.0:1775768159.942054:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768159.942059:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768159.942065:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768159.942076:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967394688/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768159.942082:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768159.942086:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768159.942089:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768159.942096:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4781 00000800:00000100:3.0:1775768159.942096:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775768159.942105:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768159.942110:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768159.942115:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768159.942121:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967394816/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775768159.942187:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768159/real 1775768159] req@ffff8f994f2e6680 x1862022967394432/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768159.942331:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967394432/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768159.942346:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768159.942352:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768159.942357:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967394432:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775768159.942435:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967394560:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775768159.942461:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768159.942474:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768159/real 1775768159] req@ffff8f994f2e4000 x1862022967394560/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768159.942485:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967394560/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768159.942544:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768159.942549:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768159.942553:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967394560:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775768159.942560:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967394688:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768159.942582:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768159.942593:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768159/real 1775768159] req@ffff8f994f2e5f80 x1862022967394688/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768159.942605:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967394688/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768159.942613:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768159.942616:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768159.942619:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967394688:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775768159.942625:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967394816:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768159.942645:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768159.942656:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768159/real 1775768159] req@ffff8f994f2e6a00 x1862022967394816/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768159.942686:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967394816/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768159.942694:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768159.942697:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768159.942715:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967394816:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768160.837829:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768160.837837:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768161.861827:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768161.861873:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768162.885842:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768162.885915:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768163.909829:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768163.909882:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768164.933877:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768164.933953:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768165.061854:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768165.061869:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768165.061875:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768165.061888:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4786 00000100:00080000:1.0:1775768165.061903:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768165.062082:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775768165.062124:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967394944/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768165.062152:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768165.062156:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768165.062158:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768165.062162:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4786 00000100:00080000:1.0:1775768165.062171:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768165.062174:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768165.062182:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768165.062190:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967395072/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768165.062198:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768165.062200:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768165.062201:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768165.062203:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4786 00000100:00080000:1.0:1775768165.062205:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768165.062208:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768165.062211:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768165.062219:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967395200/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775768165.062220:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967394944:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775768165.062231:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768165.062234:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768165.062253:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768165.062256:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4786 00000100:00080000:1.0:1775768165.062259:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768165.062261:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768165.062265:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768165.062272:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967395328/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775768165.062292:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768165.062334:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768165/real 1775768165] req@ffff8f994f2e5180 x1862022967394944/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768165.062350:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967394944/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768165.062358:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768165.062361:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768165.062364:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967394944:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775768165.062370:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967395072:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775768165.062386:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768165.062394:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768165/real 1775768165] req@ffff8f994f2e4a80 x1862022967395072/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768165.062402:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967395072/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768165.062408:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768165.062410:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768165.062412:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967395072:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775768165.062432:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967395200:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768165.062443:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768165.062468:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768165/real 1775768165] req@ffff8f994f2e7800 x1862022967395200/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768165.062477:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967395200/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768165.062490:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768165.062493:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768165.062495:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967395200:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775768165.062499:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967395328:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768165.062512:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768165.062519:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768165/real 1775768165] req@ffff8f994f2e6d80 x1862022967395328/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768165.062526:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967395328/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768165.062531:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768165.062533:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768165.062535:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967395328:192.168.202.146@tcp:8:kworker.0 00010000:00010000:1.0:1775768165.957923:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768165.958007:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775768166.981839:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768166.981890:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768168.005779:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768168.005856:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768169.029867:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768169.029947:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768170.053850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768170.053932:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768170.181869:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768170.181878:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768170.181883:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768170.181892:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4791 00000100:00080000:3.0:1775768170.181904:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768170.181910:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768170.181976:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967395456/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768170.182009:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768170.182013:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768170.182014:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768170.182018:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4791 00000100:00080000:3.0:1775768170.182022:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768170.182026:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768170.182035:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768170.182047:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967395584/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768170.182057:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967395456:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775768170.182059:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768170.182066:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768170.182067:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768170.182072:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4791 00000100:00080000:3.0:1775768170.182083:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768170.182087:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768170.182092:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768170.182104:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967395712/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768170.182113:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768170.182118:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:2.0:1775768170.182119:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775768170.182120:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768170.182139:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4791 00000100:00080000:3.0:1775768170.182144:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768170.182148:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768170.182153:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768170.182165:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768170/real 1775768170] req@ffff8f9970edfb80 x1862022967395456/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768170.182167:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967395840/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768170.182194:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967395456/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768170.182206:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768170.182210:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768170.182215:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967395456:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775768170.182235:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967395584:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775768170.182263:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768170.182273:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768170/real 1775768170] req@ffff8f9970ede300 x1862022967395584/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768170.182284:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967395584/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768170.182292:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768170.182295:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768170.182298:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967395584:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775768170.182303:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967395712:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775768170.182318:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768170.182326:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768170/real 1775768170] req@ffff8f9970edf800 x1862022967395712/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768170.182336:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967395712/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768170.182349:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768170.182352:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768170.182355:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967395712:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768170.182360:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967395840:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775768170.182375:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768170.182383:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768170/real 1775768170] req@ffff8f9970edf480 x1862022967395840/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768170.182393:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967395840/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768170.182401:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768170.182404:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768170.182407:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967395840:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768171.077824:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768171.077877:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768172.101851:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768172.101909:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768173.125886:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768173.125922:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768174.149835:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768174.149882:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768175.173848:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768175.173905:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768175.301823:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768175.301832:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768175.301837:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768175.301847:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4796 00000100:00080000:3.0:1775768175.301859:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768175.301865:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768175.301913:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967395968/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768175.301939:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768175.301942:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768175.301943:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768175.301947:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4796 00000100:00080000:3.0:1775768175.301950:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768175.301953:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768175.301962:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768175.301973:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967396096/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768175.301981:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775768175.301991:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967395968:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775768175.302109:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967396096:192.168.202.146@tcp:38:kworker.0 00000100:00080000:3.0:1775768175.302140:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768175.302144:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768175.302152:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4796 00000100:00080000:3.0:1775768175.302161:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768175.302168:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768175.302179:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768175.302209:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967396224/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768175.302222:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768175.302225:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768175.302226:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768175.302229:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4796 00000100:00080000:3.0:1775768175.302230:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768175.302232:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768175.302235:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768175.302242:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967396352/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768175.302288:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967396224:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768175.302314:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967396352:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768176.197822:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768176.197882:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768177.221830:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768177.221886:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768178.245817:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768178.245934:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775768178.374073:0:2369:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:1.0:1775768178.374121:0:2369:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775768178.374140:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775768178.374182:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775768178.374192:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775768178.374201:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:2.0:1775768178.374585:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768175/real 1775768178] req@ffff8f9970eded80 x1862022967396224/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768178.374625:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967396224/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768178.374645:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768178.374652:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768178.374657:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967396224:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768178.374666:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768175/real 1775768178] req@ffff8f9970ede680 x1862022967396352/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768178.374682:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967396352/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768178.374692:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768178.374696:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768178.374700:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967396352:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768178.374708:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768175/real 1775768178] req@ffff8f9970edd880 x1862022967396096/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768178.374724:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967396096/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768178.374733:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768178.374771:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768178.374777:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967396096:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775768178.374786:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768175/real 1775768178] req@ffff8f9970edc380 x1862022967395968/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768178.374802:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967395968/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768178.374811:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768178.374815:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768178.374819:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967395968:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775768179.269824:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768179.269914:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768180.293911:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768180.293940:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768180.421848:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768180.421859:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768180.421865:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768180.421876:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4802 00000100:00080000:3.0:1775768180.421888:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768180.421894:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768180.421928:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddc00 x1862022967396480/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768180.422000:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768180.422004:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768180.422007:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768180.422012:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4802 00000100:00080000:3.0:1775768180.422016:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768180.422021:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768180.422031:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768180.422044:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967396608/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768180.422053:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768180.422056:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768180.422058:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768180.422062:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4802 00000100:00080000:3.0:1775768180.422066:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768180.422070:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768180.422075:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768180.422087:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf100 x1862022967396736/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768180.422107:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768180.422110:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768180.422112:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768180.422116:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4802 00000100:00080000:3.0:1775768180.422138:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768180.422143:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768180.422148:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768180.422162:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967396864/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768180.422187:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967396480:192.168.202.146@tcp:250:kworker.0 00000800:00000100:2.0:1775768180.422270:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768180.422348:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768180/real 1775768180] req@ffff8f9970eddc00 x1862022967396480/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768180.422371:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddc00 x1862022967396480/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768180.422385:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768180.422390:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768180.422397:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967396480:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775768180.422414:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967396608:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775768180.422444:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768180.422458:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768180/real 1775768180] req@ffff8f9970edc000 x1862022967396608/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768180.422504:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967396608/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768180.422513:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768180.422516:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768180.422520:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967396608:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775768180.422527:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967396736:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775768180.422550:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768180.422562:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768180/real 1775768180] req@ffff8f9970edf100 x1862022967396736/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768180.422574:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf100 x1862022967396736/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768180.422582:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768180.422586:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768180.422589:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967396736:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768180.422623:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967396864:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775768180.422644:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768180.422656:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768180/real 1775768180] req@ffff8f9970edc700 x1862022967396864/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768180.422669:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967396864/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768180.422678:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768180.422682:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768180.422686:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967396864:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768181.317880:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768181.318035:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768182.341832:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768182.341891:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768183.365848:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768183.365915:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768184.389848:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768184.389918:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768185.413819:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768185.413883:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768185.541844:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768185.541862:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768185.541868:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768185.541879:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4807 00000100:00080000:3.0:1775768185.541890:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768185.541898:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768185.541935:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967396992/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768185.541959:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768185.541962:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768185.541964:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768185.541967:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4807 00000100:00080000:3.0:1775768185.541970:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768185.541973:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768185.541981:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768185.541990:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967397120/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768185.541998:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775768185.542004:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967396992:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775768185.542036:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768185.542038:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768185.542041:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4807 00000100:00080000:3.0:1775768185.542050:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768185.542053:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768185.542057:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768185.542067:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967397248/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775768185.542069:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775768185.542078:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768185.542081:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768185.542083:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768185.542088:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4807 00000100:00080000:3.0:1775768185.542101:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768185.542106:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768185.542111:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768185.542116:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768185/real 1775768185] req@ffff8f9970edce00 x1862022967396992/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768185.542126:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf100 x1862022967397376/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768185.542142:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967396992/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768185.542155:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768185.542161:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768185.542165:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967396992:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775768185.542186:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967397120:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775768185.542208:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768185.542222:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768185/real 1775768185] req@ffff8f9970edd500 x1862022967397120/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768185.542235:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967397120/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768185.542246:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768185.542249:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768185.542253:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967397120:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775768185.542259:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967397248:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775768185.542280:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768185.542291:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768185/real 1775768185] req@ffff8f9970edc700 x1862022967397248/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768185.542305:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967397248/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768185.542314:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768185.542318:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768185.542321:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967397248:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768185.542328:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967397376:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775768185.542346:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768185.542357:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768185/real 1775768185] req@ffff8f9970edf100 x1862022967397376/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768185.542369:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf100 x1862022967397376/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768185.542378:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768185.542381:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768185.542386:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967397376:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768186.437834:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768186.437873:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768187.461878:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768187.461908:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768188.485833:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768188.485887:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768189.509837:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768189.509896:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768190.533853:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768190.533915:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768190.661837:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768190.661845:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768190.661849:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768190.661856:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4812 00000100:00080000:3.0:1775768190.661865:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768190.661869:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768190.661899:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967397504/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768190.661949:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768190.661952:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768190.661953:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768190.661956:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4812 00000100:00080000:3.0:1775768190.661958:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768190.661961:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768190.661967:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768190.661974:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddc00 x1862022967397632/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768190.661981:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768190.661983:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768190.661984:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768190.661986:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4812 00000100:00080000:3.0:1775768190.661988:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768190.661990:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768190.661993:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768190.661997:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967397504:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775768190.662001:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967397760/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768190.662008:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768190.662010:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768190.662010:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768190.662012:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4812 00000100:00080000:3.0:1775768190.662024:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768190.662027:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768190.662030:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768190.662036:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967397888/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775768190.662051:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768190.662093:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768190/real 1775768190] req@ffff8f9970edc000 x1862022967397504/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768190.662106:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967397504/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768190.662114:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768190.662117:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768190.662120:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967397504:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775768190.662125:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967397632:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775768190.662138:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768190.662145:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768190/real 1775768190] req@ffff8f9970eddc00 x1862022967397632/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768190.662151:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddc00 x1862022967397632/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768190.662156:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768190.662158:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768190.662160:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967397632:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775768190.662176:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967397760:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775768190.662186:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768190.662191:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768190/real 1775768190] req@ffff8f9970edc380 x1862022967397760/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768190.662198:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967397760/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768190.662202:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768190.662203:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768190.662205:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967397760:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768190.662208:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967397888:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775768190.662215:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768190.662220:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768190/real 1775768190] req@ffff8f9970edd880 x1862022967397888/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768190.662226:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967397888/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768190.662231:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768190.662232:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768190.662234:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967397888:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768191.557827:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768191.557883:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768192.581863:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768192.581938:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768193.605843:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768193.605905:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768194.629880:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768194.629904:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768195.653856:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768195.653913:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768195.781905:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768195.781918:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768195.781924:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768195.781937:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4817 00000100:00080000:3.0:1775768195.781950:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768195.781957:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768195.782010:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967398016/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768195.782045:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768195.782050:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768195.782052:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768195.782056:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4817 00000100:00080000:3.0:1775768195.782060:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768195.782064:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768195.782075:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768195.782099:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967398144/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768195.782111:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775768195.782117:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967398016:192.168.202.146@tcp:250:kworker.0 00000800:00000100:2.0:1775768195.782195:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768195.782263:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768195/real 1775768195] req@ffff8f9970ede680 x1862022967398016/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768195.782265:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768195.782272:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768195.782287:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4817 00000100:00080000:3.0:1775768195.782298:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775768195.782302:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967398016/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768195.782310:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768195.782340:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1775768195.782344:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768195.782350:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768195.782355:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967398016:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775768195.782373:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967398272/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768195.782388:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768195.782390:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1775768195.782393:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967398144:192.168.202.146@tcp:38:kworker.0 00000100:00080000:3.0:1775768195.782395:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768195.782405:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4817 00000100:00080000:3.0:1775768195.782410:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768195.782414:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768195.782419:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775768195.782428:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768195.782429:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967398400/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768195.782442:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768195/real 1775768195] req@ffff8f9970eded80 x1862022967398144/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768195.782452:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967398144/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768195.782458:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768195.782461:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768195.782463:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967398144:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775768195.782469:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967398272:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775768195.782490:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768195.782497:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768195/real 1775768195] req@ffff8f9970edf480 x1862022967398272/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768195.782506:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967398272/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768195.782514:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768195.782516:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768195.782518:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967398272:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768195.782528:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967398400:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775768195.782543:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768195.782552:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768195/real 1775768195] req@ffff8f9970edf800 x1862022967398400/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768195.782559:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967398400/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768195.782563:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768195.782565:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768195.782567:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967398400:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768196.677836:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768196.677890:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768197.701826:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768197.701910:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768198.725840:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768198.725890:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768199.749890:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768199.749997:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768200.773854:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768200.773919:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768200.901908:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768200.901920:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768200.901925:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768200.901939:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4822 00000100:00080000:3.0:1775768200.901962:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768200.901972:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768200.902025:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967398528/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768200.902321:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768200.902327:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768200.902330:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768200.902341:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4822 00000100:00080000:3.0:1775768200.902355:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768200.902362:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768200.902376:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768200.902414:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967398656/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768200.902424:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768200.902427:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768200.902429:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768200.902432:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4822 00000100:00080000:3.0:1775768200.902435:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768200.902439:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768200.902443:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768200.902457:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967398784/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768200.902465:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768200.902468:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768200.902470:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768200.902473:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4822 00000100:00080000:3.0:1775768200.902511:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768200.902515:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768200.902521:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768200.902534:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967398912/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768200.902640:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967398528:192.168.202.146@tcp:250:kworker.0 00000800:00000100:2.0:1775768200.902703:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768200.902783:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768200/real 1775768200] req@ffff8f9970ede300 x1862022967398528/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768200.902797:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967398528/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768200.902804:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768200.902807:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768200.902813:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967398528:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775768200.902819:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967398656:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775768200.902838:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768200.902845:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768200/real 1775768200] req@ffff8f9970edfb80 x1862022967398656/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768200.902852:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967398656/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768200.902857:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768200.902859:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768200.902860:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967398656:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775768200.902864:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967398784:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775768200.902874:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768200.902879:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768200/real 1775768200] req@ffff8f9970edca80 x1862022967398784/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768200.902895:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967398784/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768200.902903:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768200.902905:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768200.902908:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967398784:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768200.902913:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967398912:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775768200.902931:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768200.902941:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768200/real 1775768200] req@ffff8f9970edea00 x1862022967398912/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768200.902956:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967398912/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768200.902961:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768200.902964:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768200.902966:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967398912:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768201.797915:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768201.797950:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768202.821841:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768202.821913:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768203.845850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768203.845887:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768204.869859:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768204.869897:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768205.893888:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768205.893947:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768206.021845:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768206.021856:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768206.021861:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768206.021872:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4827 00000100:00080000:3.0:1775768206.021883:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768206.021892:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768206.021928:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd180 x1862022967399040/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768206.021957:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768206.021961:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768206.021963:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768206.021968:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4827 00000100:00080000:3.0:1775768206.021972:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768206.021976:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768206.021985:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768206.021998:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddf80 x1862022967399168/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768206.022011:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768206.022041:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768206.022044:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768206.022049:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4827 00000100:00080000:3.0:1775768206.022053:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768206.022057:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768206.022064:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768206.022079:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967399296/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768206.022097:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967399040:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775768206.022100:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768206.022106:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768206.022108:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768206.022113:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4827 00000100:00080000:3.0:1775768206.022117:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768206.022128:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768206.022135:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768206.022152:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967399424/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775768206.022277:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768206.022381:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768206/real 1775768206] req@ffff8f9970edd180 x1862022967399040/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768206.022420:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd180 x1862022967399040/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768206.022440:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768206.022448:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768206.022456:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967399040:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775768206.022469:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967399168:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775768206.022504:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768206.022520:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768206/real 1775768206] req@ffff8f9970eddf80 x1862022967399168/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768206.022534:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddf80 x1862022967399168/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768206.022547:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768206.022553:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768206.022558:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967399168:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775768206.022602:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967399296:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775768206.022642:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768206.022663:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768206/real 1775768206] req@ffff8f9970edea00 x1862022967399296/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768206.022684:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967399296/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768206.022707:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768206.022713:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768206.022719:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967399296:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768206.022729:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967399424:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775768206.022832:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768206.022850:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768206/real 1775768206] req@ffff8f9970edca80 x1862022967399424/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768206.022870:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967399424/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768206.022884:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768206.022890:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768206.022895:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967399424:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768206.917843:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768206.917897:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768207.941897:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768207.941959:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768208.965893:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768208.965923:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768209.989844:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768209.989947:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768211.013821:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768211.013883:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768211.141867:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768211.141875:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768211.141879:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768211.141887:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4832 00000100:00080000:1.0:1775768211.141900:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768211.141905:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775768211.141937:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967399552/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768211.141960:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768211.141962:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768211.141963:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768211.141966:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4832 00000100:00080000:1.0:1775768211.141968:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768211.141970:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768211.141978:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768211.141986:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967399680/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768211.141993:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775768211.141994:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967399552:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775768211.141996:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768211.141999:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768211.142005:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4832 00000100:00080000:1.0:1775768211.142007:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768211.142009:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768211.142045:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768211.142052:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967399808/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768211.142058:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768211.142060:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768211.142062:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1775768211.142066:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775768211.142069:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4832 00000100:00080000:1.0:1775768211.142072:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768211.142075:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768211.142079:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768211.142086:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967399936/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768211.142108:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768211/real 1775768211] req@ffff8f994f2e6d80 x1862022967399552/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768211.142122:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967399552/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768211.142134:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768211.142138:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768211.142143:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967399552:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775768211.142164:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967399680:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775768211.142194:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768211.142203:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768211/real 1775768211] req@ffff8f994f2e7800 x1862022967399680/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768211.142211:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967399680/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768211.142216:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768211.142219:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768211.142222:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967399680:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775768211.142228:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967399808:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775768211.142246:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768211.142256:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768211/real 1775768211] req@ffff8f994f2e4a80 x1862022967399808/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768211.142267:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967399808/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768211.142275:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768211.142278:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768211.142281:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967399808:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768211.142287:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967399936:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775768211.142303:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768211.142311:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768211/real 1775768211] req@ffff8f994f2e5180 x1862022967399936/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768211.142321:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967399936/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768211.142328:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768211.142331:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768211.142333:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967399936:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768212.037848:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768212.037899:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768213.061824:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768213.061875:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768214.085842:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768214.085900:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768215.109812:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768215.109856:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768216.133911:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768216.133938:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768216.261847:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768216.261854:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768216.261858:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768216.261867:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4837 00000100:00080000:3.0:1775768216.261876:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768216.261881:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768216.261912:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967400064/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768216.261961:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768216.261963:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768216.261965:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768216.261968:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4837 00000100:00080000:3.0:1775768216.261970:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768216.261972:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768216.261979:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768216.261987:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967400192/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768216.261994:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768216.261997:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768216.261999:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768216.262003:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4837 00000100:00080000:3.0:1775768216.262006:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768216.262011:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775768216.262013:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967400064:192.168.202.146@tcp:250:kworker.0 00000008:00100000:3.0:1775768216.262016:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768216.262029:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967400320/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768216.262039:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768216.262042:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768216.262044:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768216.262047:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4837 00000800:00000100:2.0:1775768216.262066:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775768216.262069:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768216.262074:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768216.262079:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768216.262088:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967400448/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768216.262128:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768216/real 1775768216] req@ffff8f9970edfb80 x1862022967400064/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768216.262144:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967400064/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768216.262153:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768216.262156:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768216.262165:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967400064:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775768216.262175:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967400192:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775768216.262199:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768216.262210:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768216/real 1775768216] req@ffff8f9970ede300 x1862022967400192/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768216.262220:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967400192/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768216.262227:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768216.262230:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768216.262233:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967400192:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775768216.262254:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967400320:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775768216.262285:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768216.262293:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768216/real 1775768216] req@ffff8f9970edf800 x1862022967400320/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768216.262303:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967400320/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768216.262309:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768216.262312:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768216.262315:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967400320:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768216.262320:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967400448:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775768216.262336:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768216.262346:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768216/real 1775768216] req@ffff8f9970edf480 x1862022967400448/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768216.262354:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967400448/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768216.262359:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768216.262361:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768216.262364:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967400448:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768217.157846:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768217.157919:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768218.181843:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768218.181889:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768219.205834:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768219.205891:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768220.229863:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768220.229918:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768221.253885:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768221.253948:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768221.381825:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768221.381832:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768221.381836:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768221.381843:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4842 00000100:00080000:3.0:1775768221.381852:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768221.381857:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768221.381887:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967400576/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768221.381912:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768221.381914:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768221.381915:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768221.381918:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4842 00000100:00080000:3.0:1775768221.381920:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768221.381922:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768221.381929:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768221.381936:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967400704/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768221.381942:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768221.381963:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768221.381964:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768221.381971:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4842 00000100:00080000:3.0:1775768221.381974:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768221.381977:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768221.381980:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768221.381988:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967400576:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775768221.381991:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967400832/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768221.382008:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768221.382011:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768221.382013:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768221.382017:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4842 00000100:00080000:3.0:1775768221.382020:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768221.382024:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768221.382029:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768221.382042:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967400960/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775768221.382097:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768221.382152:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768221/real 1775768221] req@ffff8f9970eded80 x1862022967400576/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768221.382178:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967400576/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768221.382189:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768221.382194:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768221.382198:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967400576:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775768221.382208:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967400704:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775768221.382233:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768221.382245:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768221/real 1775768221] req@ffff8f9970ede680 x1862022967400704/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768221.382258:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967400704/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768221.382271:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768221.382275:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768221.382278:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967400704:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775768221.382308:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967400832:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775768221.382329:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768221.382339:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768221/real 1775768221] req@ffff8f9970edd880 x1862022967400832/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768221.382350:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967400832/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768221.382358:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768221.382361:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768221.382364:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967400832:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768221.382370:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967400960:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775768221.382387:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768221.382397:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768221/real 1775768221] req@ffff8f9970edc380 x1862022967400960/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768221.382407:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967400960/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768221.382420:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768221.382423:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768221.382426:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967400960:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768222.277808:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768222.277884:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768223.301913:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768223.301931:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768224.325821:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768224.325884:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768225.349810:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768225.349863:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768226.373832:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768226.373877:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768226.501934:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768226.501943:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768226.501947:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768226.501958:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4848 00000100:00080000:1.0:1775768226.501968:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768226.501975:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775768226.502009:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967401088/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768226.502088:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768226.502091:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768226.502092:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768226.502096:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4848 00000100:00080000:1.0:1775768226.502100:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768226.502104:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768226.502113:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768226.502127:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967401216/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768226.502135:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967401088:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775768226.502140:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768226.502149:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768226.502151:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768226.502155:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4848 00000100:00080000:1.0:1775768226.502158:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768226.502162:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768226.502167:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768226.502177:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967401344/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768226.502186:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768226.502190:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768226.502191:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768226.502195:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4848 00000800:00000100:2.0:1775768226.502202:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775768226.502213:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768226.502218:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768226.502223:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768226.502235:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967401472/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768226.502240:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768226/real 1775768226] req@ffff8f994f2e6a00 x1862022967401088/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768226.502270:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967401088/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768226.502279:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768226.502283:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768226.502287:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967401088:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775768226.502309:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967401216:192.168.202.146@tcp:38:kworker.0 00000800:00000100:2.0:1775768226.502330:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768226.502341:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768226/real 1775768226] req@ffff8f994f2e5f80 x1862022967401216/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768226.502355:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967401216/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768226.502360:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768226.502362:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768226.502364:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967401216:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775768226.502368:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967401344:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775768226.502381:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768226.502387:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768226/real 1775768226] req@ffff8f994f2e4000 x1862022967401344/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768226.502394:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967401344/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768226.502402:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768226.502405:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768226.502408:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967401344:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768226.502413:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967401472:192.168.202.146@tcp:8:kworker.0 00000800:00000100:2.0:1775768226.502430:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768226.502440:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768226/real 1775768226] req@ffff8f994f2e6680 x1862022967401472/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768226.502451:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967401472/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768226.502458:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768226.502459:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:2.0:1775768226.502462:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967401472:192.168.202.146@tcp:8:kworker.0 00010000:00010000:0.0:1775768227.397900:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768227.397933:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775768228.421866:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768228.421930:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768229.445805:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768229.445846:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768230.469879:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768230.469895:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768231.493841:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768231.493906:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768231.621838:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768231.621852:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768231.621855:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768231.621863:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4853 00000100:00080000:0.0:1775768231.621872:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768231.621878:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775768231.621906:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967401600/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768231.621931:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768231.621933:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768231.621934:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768231.621938:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4853 00000100:00080000:0.0:1775768231.621940:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768231.621943:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768231.621950:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768231.621961:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967401728/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775768231.621962:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967401600:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775768231.621986:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768231.621988:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768231.621989:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768231.621992:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4853 00000100:00080000:0.0:1775768231.621995:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768231.621998:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768231.622002:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768231.622013:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967401856/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768231.622019:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768231.622021:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775768231.622022:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768231.622024:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4853 00000100:00080000:0.0:1775768231.622032:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768231.622035:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768231.622037:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775768231.622043:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768231.622046:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967401984/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775768231.622088:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768231/real 1775768231] req@ffff8f9970a77800 x1862022967401600/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768231.622163:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967401600/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768231.622210:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768231.622214:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768231.622302:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967401600:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775768231.622320:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967401728:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775768231.622339:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768231.622384:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768231/real 1775768231] req@ffff8f9970a76d80 x1862022967401728/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768231.622400:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967401728/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768231.622405:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768231.622407:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768231.622409:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967401728:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775768231.622413:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967401856:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768231.622432:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768231.622442:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768231/real 1775768231] req@ffff8f9970a74700 x1862022967401856/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768231.622452:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967401856/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768231.622460:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768231.622463:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768231.622466:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967401856:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775768231.622485:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967401984:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768231.622500:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768231.622506:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768231/real 1775768231] req@ffff8f9970a74380 x1862022967401984/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768231.622513:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967401984/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768231.622518:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768231.622519:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768231.622521:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967401984:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768232.517827:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768232.517879:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768233.541838:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768233.541891:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768234.565836:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768234.565889:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768235.589844:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768235.589892:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768236.613850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768236.613917:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768236.741901:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768236.741914:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768236.741934:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768236.741946:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4858 00000100:00080000:1.0:1775768236.741958:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768236.741965:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775768236.742018:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967402112/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768236.742051:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768236.742054:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768236.742056:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768236.742060:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4858 00000100:00080000:1.0:1775768236.742064:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768236.742067:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768236.742077:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768236.742090:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967402240/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775768236.742104:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967402112:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775768236.742110:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768236.742117:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768236.742119:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768236.742124:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4858 00000100:00080000:1.0:1775768236.742128:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768236.742131:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768236.742137:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768236.742150:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967402368/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768236.742158:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768236.742169:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768236.742171:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768236.742174:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4858 00000100:00080000:1.0:1775768236.742178:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768236.742182:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:3.0:1775768236.742185:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:1.0:1775768236.742188:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768236.742201:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967402496/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775768236.742245:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768236/real 1775768236] req@ffff8f994f2e4e00 x1862022967402112/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768236.742269:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967402112/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768236.742280:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768236.742284:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768236.742288:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967402112:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775768236.742320:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967402240:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775768236.742343:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768236.742355:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768236/real 1775768236] req@ffff8f994f2e7b80 x1862022967402240/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768236.742534:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967402240/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768236.742546:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768236.742550:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768236.742554:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967402240:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775768236.742566:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967402368:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768236.742622:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768236.742647:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768236/real 1775768236] req@ffff8f994f2e4700 x1862022967402368/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768236.742655:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967402368/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768236.742660:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768236.742662:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768236.742664:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967402368:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775768236.742668:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967402496:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768236.742680:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768236.742685:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768236/real 1775768236] req@ffff8f994f2e6300 x1862022967402496/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768236.742692:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967402496/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768236.742712:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768236.742714:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768236.742716:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967402496:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768237.637926:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768237.637966:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768238.661886:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768238.661967:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775768239.685828:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768239.685903:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768240.709854:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768240.709924:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768241.733782:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768241.733863:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768241.861856:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768241.861865:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768241.861899:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768241.861910:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4863 00000100:00080000:1.0:1775768241.861921:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768241.861927:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775768241.861960:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967402624/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768241.861987:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768241.861991:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768241.861992:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768241.861996:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4863 00000100:00080000:1.0:1775768241.862000:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768241.862004:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768241.862012:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768241.862024:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967402752/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768241.862032:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768241.862034:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768241.862036:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775768241.862039:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967402624:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775768241.862040:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4863 00000100:00080000:1.0:1775768241.862044:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768241.862050:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768241.862053:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768241.862060:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967402880/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768241.862097:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768241.862099:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768241.862101:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768241.862103:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4863 00000100:00080000:1.0:1775768241.862106:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768241.862107:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768241.862110:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768241.862116:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967403008/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775768241.862156:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967402752:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775768241.862184:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967402880:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775768241.862205:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967403008:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768242.757848:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768242.757913:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768243.781833:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768243.781893:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768244.805943:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768244.806195:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000400:00000100:1.0:1775768244.934089:0:2367:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:1.0:1775768244.934130:0:2367:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775768244.934146:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775768244.934190:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775768244.934198:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:1.0:1775768244.934206:0:2367:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:3.0:1775768244.934271:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768241/real 1775768244] req@ffff8f994f2e4700 x1862022967402752/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768244.934295:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967402752/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768244.934307:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768244.934311:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768244.934315:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967402752:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775768244.934322:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768241/real 1775768244] req@ffff8f994f2e7b80 x1862022967402880/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768244.934334:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967402880/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768244.934341:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768244.934365:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768244.934369:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967402880:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775768244.934376:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768241/real 1775768244] req@ffff8f994f2e4e00 x1862022967403008/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768244.934387:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967403008/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768244.934394:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768244.934397:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768244.934400:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967403008:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775768244.934407:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768241/real 1775768244] req@ffff8f994f2e6300 x1862022967402624/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768244.934417:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967402624/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768244.934423:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768244.934426:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768244.934429:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967402624:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775768245.829847:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768245.829943:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768246.853886:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768246.853943:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768246.981999:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768246.982009:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768246.982013:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768246.982022:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4868 00000100:00080000:0.0:1775768246.982033:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768246.982041:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775768246.982072:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967403136/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768246.982104:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768246.982108:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768246.982110:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768246.982114:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4868 00000100:00080000:0.0:1775768246.982118:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768246.982121:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768246.982131:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768246.982156:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967403264/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775768246.982162:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967403136:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775768246.982166:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768246.982173:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768246.982174:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768246.982179:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4868 00000100:00080000:0.0:1775768246.982183:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768246.982188:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768246.982193:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768246.982207:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967403392/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768246.982216:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768246.982219:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775768246.982221:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768246.982224:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4868 00000800:00000100:3.0:1775768246.982245:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775768246.982253:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768246.982258:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768246.982264:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768246.982276:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967403520/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775768246.982420:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768246/real 1775768246] req@ffff8f9970a76a00 x1862022967403136/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768246.982440:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967403136/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768246.982469:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768246.982474:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768246.982478:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967403136:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775768246.982512:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967403264:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775768246.982901:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768246.982916:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768246/real 1775768246] req@ffff8f9970a77100 x1862022967403264/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768246.982928:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967403264/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768246.982936:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768246.982939:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768246.982942:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967403264:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775768246.982948:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967403392:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768246.982967:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768246.982981:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768246/real 1775768246] req@ffff8f9970a75c00 x1862022967403392/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768246.982990:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967403392/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768246.982998:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768246.983001:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768246.983003:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967403392:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775768246.983029:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967403520:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768246.983048:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768246.983058:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768246/real 1775768246] req@ffff8f9970a77480 x1862022967403520/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768246.983070:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967403520/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768246.983081:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768246.983084:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768246.983088:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967403520:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768247.877872:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768247.877928:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768248.901955:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768248.902000:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775768249.925848:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768249.925901:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768250.949877:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768250.949945:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768251.973952:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768251.974098:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00080000:2.0:1775768252.101911:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768252.101924:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768252.101930:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768252.101943:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4873 00000100:00080000:2.0:1775768252.101960:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768252.101969:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775768252.102017:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3480 x1862022967403648/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768252.102042:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768252.102045:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768252.102046:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768252.102049:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4873 00000100:00080000:2.0:1775768252.102052:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768252.102054:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768252.102064:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768252.102074:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1180 x1862022967403776/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768252.102082:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768252.102084:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775768252.102085:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768252.102087:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4873 00000100:00080000:2.0:1775768252.102090:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768252.102095:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768252.102096:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967403648:192.168.202.146@tcp:250:kworker.0 00000008:00100000:2.0:1775768252.102100:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768252.102115:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2300 x1862022967403904/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768252.102124:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768252.102128:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775768252.102130:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768252.102133:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4873 00000100:00080000:2.0:1775768252.102137:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768252.102140:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768252.102145:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768252.102157:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3100 x1862022967404032/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775768252.102172:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768252.102221:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768252/real 1775768252] req@ffff8f99703d3480 x1862022967403648/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768252.102235:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3480 x1862022967403648/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768252.102243:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768252.102247:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768252.102250:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967403648:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775768252.102272:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967403776:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775768252.102286:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768252.102293:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768252/real 1775768252] req@ffff8f99703d1180 x1862022967403776/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768252.102422:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1180 x1862022967403776/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768252.102435:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768252.102439:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768252.102444:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967403776:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775768252.102456:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967403904:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768252.102510:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768252.102527:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768252/real 1775768252] req@ffff8f99703d2300 x1862022967403904/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768252.102535:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2300 x1862022967403904/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768252.102540:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768252.102542:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768252.102544:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967403904:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775768252.102548:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967404032:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768252.102558:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768252.102563:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768252/real 1775768252] req@ffff8f99703d3100 x1862022967404032/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768252.102570:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3100 x1862022967404032/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768252.102592:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768252.102594:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768252.102596:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967404032:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768252.997881:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768252.997965:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768254.021858:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768254.022591:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768255.045891:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768255.045954:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768256.069848:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768256.069905:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768257.093883:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768257.093972:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775768257.221862:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768257.221872:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768257.221882:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768257.221891:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4878 00000100:00080000:2.0:1775768257.221901:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768257.221906:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775768257.221945:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2d80 x1862022967404160/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768257.221979:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768257.221983:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768257.221985:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768257.221989:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4878 00000100:00080000:2.0:1775768257.221994:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768257.221997:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768257.222007:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768257.222023:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1500 x1862022967404288/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768257.222035:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768257.222048:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775768257.222050:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775768257.222051:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967404160:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775768257.222058:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4878 00000100:00080000:2.0:1775768257.222066:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768257.222070:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768257.222077:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768257.222087:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0700 x1862022967404416/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768257.222096:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768257.222099:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775768257.222100:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768257.222102:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4878 00000100:00080000:2.0:1775768257.222105:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768257.222106:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768257.222110:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768257.222116:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0e00 x1862022967404544/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775768257.222162:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768257.222236:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768257/real 1775768257] req@ffff8f99703d2d80 x1862022967404160/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768257.222259:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2d80 x1862022967404160/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768257.222271:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768257.222277:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768257.222282:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967404160:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775768257.222311:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967404288:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775768257.222334:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768257.222346:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768257/real 1775768257] req@ffff8f99703d1500 x1862022967404288/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768257.222356:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1500 x1862022967404288/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768257.222364:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768257.222368:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768257.222371:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967404288:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775768257.222574:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967404416:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768257.222628:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768257.222650:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768257/real 1775768257] req@ffff8f99703d0700 x1862022967404416/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768257.222666:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0700 x1862022967404416/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768257.222675:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768257.222678:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768257.222682:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967404416:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775768257.222688:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967404544:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768257.222704:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768257.222714:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768257/real 1775768257] req@ffff8f99703d0e00 x1862022967404544/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768257.222764:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0e00 x1862022967404544/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768257.222773:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768257.222776:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768257.222779:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967404544:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768258.117865:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768258.117926:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768259.141930:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768259.141937:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768260.165833:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768260.165933:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768261.189828:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768261.189892:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768262.213841:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768262.213894:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768262.341823:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768262.341832:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768262.341836:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768262.341846:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4883 00000100:00080000:0.0:1775768262.341856:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768262.341863:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775768262.341899:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967404672/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768262.341925:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768262.341929:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768262.341930:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768262.341935:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4883 00000100:00080000:0.0:1775768262.341959:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768262.341964:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768262.341971:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967404672:192.168.202.146@tcp:250:kworker.0 00000008:00100000:0.0:1775768262.341973:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768262.341986:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967404800/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768262.341994:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768262.341996:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768262.341997:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768262.342000:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4883 00000100:00080000:0.0:1775768262.342002:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768262.342005:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768262.342007:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768262.342014:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967404928/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768262.342020:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768262.342022:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775768262.342023:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768262.342025:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4883 00000100:00080000:0.0:1775768262.342027:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768262.342028:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768262.342031:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775768262.342035:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768262.342042:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967405056/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775768262.342085:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768262/real 1775768262] req@ffff8f9970a75180 x1862022967404672/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768262.342120:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967404672/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768262.342129:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768262.342133:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768262.342137:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967404672:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775768262.342157:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967404800:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775768262.342179:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768262.342190:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768262/real 1775768262] req@ffff8f9970a76300 x1862022967404800/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768262.342202:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967404800/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768262.342209:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768262.342212:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768262.342215:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967404800:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775768262.342221:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967404928:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768262.342238:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768262.342247:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768262/real 1775768262] req@ffff8f9970a75500 x1862022967404928/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768262.342258:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967404928/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768262.342285:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768262.342289:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768262.342293:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967404928:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775768262.342298:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967405056:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768262.342316:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768262.342326:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768262/real 1775768262] req@ffff8f9970a77480 x1862022967405056/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768262.342337:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967405056/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768262.342343:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768262.342346:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768262.342349:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967405056:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768263.237816:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768263.237956:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768264.261831:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768264.261887:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768265.285810:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768265.285908:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768266.309856:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768266.309858:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768267.333826:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768267.333882:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775768267.462491:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768267.462500:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768267.462506:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768267.462518:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4889 00000100:00080000:2.0:1775768267.462532:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768267.462540:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775768267.462597:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3800 x1862022967405184/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768267.462646:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768267.462650:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768267.462652:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768267.462659:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4889 00000100:00080000:2.0:1775768267.462663:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768267.462668:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768267.462679:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768267.462697:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0380 x1862022967405312/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768267.462698:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967405184:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775768267.462715:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768267.462718:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775768267.462720:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768267.462724:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4889 00000100:00080000:2.0:1775768267.462728:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768267.462733:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768267.462779:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768267.462793:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0000 x1862022967405440/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768267.462797:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775768267.462807:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768267.462811:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775768267.462813:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768267.462817:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4889 00000100:00080000:2.0:1775768267.462822:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768267.462825:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768267.462831:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768267.462842:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768267/real 1775768267] req@ffff8f99703d3800 x1862022967405184/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768267.462850:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2680 x1862022967405568/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768267.462864:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3800 x1862022967405184/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768267.462875:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768267.462880:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768267.462883:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967405184:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775768267.462902:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967405312:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775768267.462919:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768267.462928:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768267/real 1775768267] req@ffff8f99703d0380 x1862022967405312/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768267.462936:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0380 x1862022967405312/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768267.462941:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768267.462944:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768267.462948:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967405312:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775768267.462996:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967405440:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768267.463018:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768267.463027:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768267/real 1775768267] req@ffff8f99703d0000 x1862022967405440/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768267.463039:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0000 x1862022967405440/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768267.463056:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768267.463059:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768267.463062:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967405440:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775768267.463068:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967405568:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768267.463086:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768267.463096:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768267/real 1775768267] req@ffff8f99703d2680 x1862022967405568/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768267.463103:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2680 x1862022967405568/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768267.463108:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768267.463109:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768267.463111:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967405568:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768268.357830:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768268.357884:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768269.381851:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768269.381908:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768270.405879:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768270.405945:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768271.429824:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768271.429884:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768272.453868:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768272.453958:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775768272.581873:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768272.581881:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768272.581886:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768272.581895:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4894 00000100:00080000:2.0:1775768272.581906:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768272.581912:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775768272.581955:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2680 x1862022967405696/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768272.581987:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768272.581994:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768272.581996:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768272.581999:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4894 00000100:00080000:2.0:1775768272.582002:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768272.582004:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768272.582014:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768272.582024:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0000 x1862022967405824/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768272.582032:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768272.582034:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775768272.582035:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768272.582038:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4894 00000100:00080000:2.0:1775768272.582040:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768272.582043:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768272.582047:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768272.582053:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0380 x1862022967405952/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768272.582054:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967405696:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775768272.582059:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768272.582061:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775768272.582062:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768272.582064:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4894 00000100:00080000:2.0:1775768272.582065:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768272.582067:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768272.582069:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768272.582074:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3800 x1862022967406080/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768272.582271:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768272.582313:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768272/real 1775768272] req@ffff8f99703d2680 x1862022967405696/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768272.582326:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2680 x1862022967405696/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768272.582334:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768272.582337:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768272.582341:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967405696:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775768272.582363:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967405824:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775768272.582376:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768272.582383:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768272/real 1775768272] req@ffff8f99703d0000 x1862022967405824/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768272.582390:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0000 x1862022967405824/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768272.582394:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768272.582396:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768272.582398:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967405824:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775768272.582402:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967405952:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768272.582411:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768272.582416:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768272/real 1775768272] req@ffff8f99703d0380 x1862022967405952/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768272.582443:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0380 x1862022967405952/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768272.582447:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768272.582449:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768272.582451:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967405952:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775768272.582455:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967406080:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768272.582465:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768272.582477:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768272/real 1775768272] req@ffff8f99703d3800 x1862022967406080/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768272.582484:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3800 x1862022967406080/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768272.582489:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768272.582491:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768272.582492:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967406080:192.168.202.146@tcp:8:kworker.0 00010000:00010000:0.0:1775768273.477895:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768273.477909:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775768274.501834:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768274.501904:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768275.525820:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768275.525904:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768276.549836:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768276.549886:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768277.573822:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768277.573857:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768277.701869:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768277.701875:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768277.701879:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768277.701886:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4899 00000100:00080000:1.0:1775768277.701895:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768277.701900:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775768277.701936:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967406208/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768277.701964:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768277.701967:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768277.701969:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768277.701973:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4899 00000100:00080000:1.0:1775768277.701977:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768277.701981:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768277.701989:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768277.702003:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967406336/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768277.702006:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967406208:192.168.202.146@tcp:250:kworker.0 00000100:00080000:1.0:1775768277.702021:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768277.702023:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768277.702024:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768277.702028:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4899 00000800:00000100:0.0:1775768277.702059:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775768277.702065:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768277.702069:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768277.702074:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768277.702084:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967406464/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768277.702091:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775768277.702095:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768277/real 1775768277] req@ffff8f994f2e6680 x1862022967406208/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768277.702095:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768277.702097:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768277.702104:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4899 00000100:00080000:1.0:1775768277.702109:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768277.702111:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768277.702113:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768277.702120:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967406592/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768277.702144:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967406208/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768277.702154:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768277.702159:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768277.702162:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967406208:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775768277.702183:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967406336:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775768277.702198:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768277.702206:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768277/real 1775768277] req@ffff8f994f2e4000 x1862022967406336/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768277.702214:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967406336/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768277.702219:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768277.702220:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768277.702222:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967406336:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775768277.702226:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967406464:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768277.702235:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768277.702240:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768277/real 1775768277] req@ffff8f994f2e5f80 x1862022967406464/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768277.702247:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967406464/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768277.702266:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768277.702268:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768277.702270:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967406464:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775768277.702274:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967406592:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768277.702283:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768277.702289:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768277/real 1775768277] req@ffff8f994f2e6a00 x1862022967406592/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768277.702295:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967406592/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768277.702300:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768277.702302:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768277.702304:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967406592:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768278.597834:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768278.597892:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768279.621869:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768279.621967:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768280.645891:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768280.645904:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775768281.669846:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768281.669908:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768282.693800:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768282.693885:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768282.821866:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768282.821872:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768282.821876:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768282.821884:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4904 00000100:00080000:3.0:1775768282.821894:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768282.821899:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768282.821928:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddc00 x1862022967406720/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768282.821948:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768282.821977:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768282.821979:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768282.821982:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4904 00000100:00080000:3.0:1775768282.821985:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768282.821988:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768282.821995:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768282.821997:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967406720:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775768282.822005:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967406848/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768282.822013:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768282.822015:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768282.822017:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768282.822019:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4904 00000100:00080000:3.0:1775768282.822021:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768282.822023:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768282.822026:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768282.822033:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf100 x1862022967406976/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768282.822038:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768282.822040:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768282.822042:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768282.822043:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4904 00000100:00080000:3.0:1775768282.822047:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768282.822048:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:0.0:1775768282.822058:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:3.0:1775768282.822065:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768282.822078:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967407104/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768282.822109:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768282/real 1775768282] req@ffff8f9970eddc00 x1862022967406720/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768282.822128:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddc00 x1862022967406720/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768282.822161:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768282.822165:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768282.822169:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967406720:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775768282.822189:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967406848:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775768282.822212:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768282.822224:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768282/real 1775768282] req@ffff8f9970edc000 x1862022967406848/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768282.822235:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967406848/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768282.822243:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768282.822246:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768282.822249:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967406848:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775768282.822255:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967406976:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768282.822273:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768282.822283:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768282/real 1775768282] req@ffff8f9970edf100 x1862022967406976/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768282.822293:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf100 x1862022967406976/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768282.822301:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768282.822304:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768282.822307:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967406976:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775768282.822326:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967407104:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768282.822343:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768282.822353:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768282/real 1775768282] req@ffff8f9970edc700 x1862022967407104/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768282.822364:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967407104/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768282.822372:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768282.822375:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768282.822378:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967407104:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768283.717846:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768283.718015:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768284.741941:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775768284.742050:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:3.0:1775768285.765891:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768285.765924:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775768286.789892:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768286.789915:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768287.813956:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768287.814002:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775768287.941871:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768287.941878:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768287.941882:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768287.941890:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4909 00000100:00080000:2.0:1775768287.941898:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768287.941904:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775768287.941937:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0e00 x1862022967407232/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768287.941969:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768287.941973:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768287.941975:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768287.941979:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4909 00000100:00080000:2.0:1775768287.941983:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768287.941987:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768287.941996:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768287.942006:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0700 x1862022967407360/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768287.942015:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775768287.942017:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967407232:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775768287.942021:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775768287.942025:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768287.942029:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4909 00000100:00080000:2.0:1775768287.942033:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768287.942037:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768287.942043:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768287.942055:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1500 x1862022967407488/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768287.942070:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768287.942074:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775768287.942077:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1775768287.942078:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775768287.942082:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4909 00000100:00080000:2.0:1775768287.942086:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768287.942088:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768287.942092:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768287.942101:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2d80 x1862022967407616/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768287.942124:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768287/real 1775768287] req@ffff8f99703d0e00 x1862022967407232/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768287.942138:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0e00 x1862022967407232/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768287.942148:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768287.942152:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768287.942155:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967407232:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775768287.942178:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967407360:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775768287.942210:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768287.942221:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768287/real 1775768287] req@ffff8f99703d0700 x1862022967407360/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768287.942233:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0700 x1862022967407360/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768287.942240:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768287.942243:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768287.942247:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967407360:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775768287.942252:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967407488:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768287.942271:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768287.942279:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768287/real 1775768287] req@ffff8f99703d1500 x1862022967407488/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768287.942290:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1500 x1862022967407488/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768287.942297:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768287.942299:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768287.942302:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967407488:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775768287.942308:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967407616:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768287.942326:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768287.942335:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768287/real 1775768287] req@ffff8f99703d2d80 x1862022967407616/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768287.942349:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2d80 x1862022967407616/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768287.942355:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768287.942356:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768287.942358:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967407616:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768288.837828:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768288.837943:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768289.861850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768289.861905:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768290.885836:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768290.885901:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768291.909871:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768291.909948:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768292.933848:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768292.933897:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768293.061869:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768293.061876:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768293.061879:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768293.061888:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4914 00000100:00080000:0.0:1775768293.061899:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768293.061905:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775768293.061953:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967407744/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768293.061976:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768293.061979:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768293.061980:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768293.061983:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4914 00000100:00080000:0.0:1775768293.061986:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768293.061988:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768293.061997:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768293.062011:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967407872/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768293.062020:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768293.062024:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768293.062026:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768293.062030:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4914 00000100:00080000:0.0:1775768293.062033:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768293.062038:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768293.062043:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768293.062055:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967407744:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775768293.062056:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967408000/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768293.062079:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768293.062081:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775768293.062082:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768293.062087:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4914 00000100:00080000:0.0:1775768293.062090:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768293.062092:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768293.062097:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768293.062105:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967408128/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775768293.062156:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768293.062210:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768293/real 1775768293] req@ffff8f9970a75c00 x1862022967407744/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768293.062230:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967407744/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768293.062241:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768293.062245:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768293.062250:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967407744:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775768293.062258:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967407872:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775768293.062282:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768293.062293:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768293/real 1775768293] req@ffff8f9970a77100 x1862022967407872/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768293.062301:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967407872/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768293.062307:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768293.062309:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768293.062512:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967407872:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775768293.062547:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967408000:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768293.062602:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768293.062625:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768293/real 1775768293] req@ffff8f9970a76a00 x1862022967408000/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768293.062641:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967408000/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768293.062650:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768293.062654:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768293.062657:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967408000:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775768293.062663:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967408128:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768293.062683:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768293.062701:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768293/real 1775768293] req@ffff8f9970a74380 x1862022967408128/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768293.062713:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967408128/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768293.062720:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768293.062723:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768293.062726:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967408128:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768293.957866:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768293.957941:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768294.981875:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768294.981891:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768296.005844:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768296.005931:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768297.029845:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768297.029906:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768298.053823:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768298.053874:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768298.181805:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768298.181846:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768298.181849:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768298.181857:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4919 00000100:00080000:0.0:1775768298.181866:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768298.181872:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775768298.181905:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967408256/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768298.181939:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768298.181946:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768298.181948:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768298.181954:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4919 00000100:00080000:0.0:1775768298.181958:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768298.181963:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768298.181973:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768298.181976:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967408256:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775768298.181988:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967408384/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768298.181999:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768298.182002:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768298.182004:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768298.182009:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4919 00000100:00080000:0.0:1775768298.182013:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768298.182017:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768298.182021:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768298.182033:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967408512/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775768298.182039:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775768298.182062:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768298.182065:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775768298.182066:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768298.182070:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4919 00000100:00080000:0.0:1775768298.182074:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768298.182077:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768298.182081:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768298.182092:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768298/real 1775768298] req@ffff8f9970a74700 x1862022967408256/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768298.182093:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967408640/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775768298.182127:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967408256/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768298.182135:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768298.182139:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768298.182141:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967408256:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775768298.182158:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967408384:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775768298.182173:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768298.182180:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768298/real 1775768298] req@ffff8f9970a76d80 x1862022967408384/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768298.182187:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967408384/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768298.182192:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768298.182194:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768298.182196:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967408384:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775768298.182199:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967408512:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768298.182208:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768298.182213:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768298/real 1775768298] req@ffff8f9970a77800 x1862022967408512/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768298.182219:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967408512/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768298.182238:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768298.182240:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768298.182242:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967408512:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775768298.182245:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967408640:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768298.182256:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768298.182261:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768298/real 1775768298] req@ffff8f9970a74380 x1862022967408640/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768298.182267:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967408640/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768298.182272:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768298.182274:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768298.182275:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967408640:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768299.077847:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768299.077898:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768300.101803:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768300.101905:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768301.125867:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768301.125940:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768302.149862:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768302.149903:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768303.173837:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768303.173914:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768303.301866:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768303.301879:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768303.301885:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768303.301898:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4924 00000100:00080000:0.0:1775768303.301913:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768303.301931:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775768303.301980:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967408768/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768303.302007:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768303.302010:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768303.302012:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775768303.302041:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967408768:192.168.202.146@tcp:250:kworker.0 00000800:00000100:1.0:1775768303.302109:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768303.302144:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768303/real 1775768303] req@ffff8f9970a76a00 x1862022967408768/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768303.302160:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967408768/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768303.302180:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768303.302182:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4924 00000100:00080000:1.0:1775768303.302189:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00080000:0.0:1775768303.302195:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775768303.302197:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967408768:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775768303.302203:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768303.302216:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768303.302248:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967408896/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768303.302294:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768303.302298:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768303.302300:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768303.302306:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4924 00000100:00080000:0.0:1775768303.302310:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768303.302313:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768303.302319:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768303.302329:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967409024/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768303.302337:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775768303.302341:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967408896:192.168.202.146@tcp:38:kworker.0 00000100:00080000:0.0:1775768303.302342:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775768303.302348:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768303.302358:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4924 00000100:00080000:0.0:1775768303.302362:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768303.302366:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768303.302371:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775768303.302386:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768303.302387:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967409152/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768303.302400:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768303/real 1775768303] req@ffff8f9970a77100 x1862022967408896/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768303.302414:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967408896/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768303.302420:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768303.302423:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768303.302426:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967408896:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775768303.302439:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967409024:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775768303.302457:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768303.302466:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768303/real 1775768303] req@ffff8f9970a75c00 x1862022967409024/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768303.302487:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967409024/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768303.302492:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768303.302495:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768303.302500:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967409024:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775768303.302505:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967409152:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775768303.302520:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768303.302529:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768303/real 1775768303] req@ffff8f9970a77480 x1862022967409152/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768303.302541:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967409152/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768303.302547:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768303.302549:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768303.302551:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967409152:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768304.197848:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768304.197908:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768305.221855:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768305.221912:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768306.245852:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768306.245913:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768307.269874:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768307.269965:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768308.293840:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768308.293898:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768308.421869:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768308.421882:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768308.421887:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768308.421900:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4930 00000100:00080000:0.0:1775768308.421914:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768308.421921:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775768308.421963:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967409280/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768308.421987:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768308.421991:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768308.421993:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768308.421997:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4930 00000100:00080000:0.0:1775768308.422000:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768308.422004:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768308.422014:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768308.422027:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967409408/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768308.422034:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967409280:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775768308.422036:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768308.422039:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768308.422041:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768308.422045:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4930 00000100:00080000:0.0:1775768308.422048:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768308.422051:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768308.422056:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768308.422063:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967409536/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768308.422072:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768308.422074:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775768308.422076:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768308.422079:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4930 00000100:00080000:0.0:1775768308.422098:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768308.422101:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768308.422105:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768308.422123:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967409664/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768308.422158:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967409408:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775768308.422314:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967409536:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775768308.422331:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967409664:192.168.202.146@tcp:8:kworker.0 00010000:00010000:0.0:1775768309.317930:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768309.317965:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775768310.341867:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768310.341871:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768311.365896:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768311.365957:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775768311.493974:0:2369:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:3.0:1775768311.494010:0:2369:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775768311.494028:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:3.0:1775768311.494077:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:3.0:1775768311.494100:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:3.0:1775768311.494108:0:2369:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:1.0:1775768311.494221:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768308/real 1775768311] req@ffff8f9970a76300 x1862022967409408/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768311.494250:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967409408/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768311.494261:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768311.494266:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768311.494269:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967409408:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775768311.494278:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768308/real 1775768311] req@ffff8f9970a75180 x1862022967409536/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768311.494289:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967409536/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768311.494297:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768311.494301:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768311.494304:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967409536:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775768311.494310:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768308/real 1775768311] req@ffff8f9970a77480 x1862022967409664/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768311.494334:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967409664/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768311.494342:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768311.494346:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768311.494350:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967409664:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775768311.494356:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768308/real 1775768311] req@ffff8f9970a75500 x1862022967409280/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768311.494369:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967409280/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768311.494376:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768311.494379:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768311.494383:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967409280:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775768312.389838:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768312.389901:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768313.413838:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768313.413896:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768313.541893:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768313.541901:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768313.541904:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768313.541913:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4935 00000100:00080000:0.0:1775768313.541924:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768313.541931:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775768313.541963:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967409792/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768313.541992:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768313.541996:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768313.541998:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768313.542002:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4935 00000100:00080000:0.0:1775768313.542006:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768313.542009:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768313.542018:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768313.542030:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967409920/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768313.542039:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768313.542042:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768313.542043:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768313.542054:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4935 00000100:00080000:0.0:1775768313.542058:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768313.542062:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768313.542067:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768313.542077:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967410048/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768313.542086:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768313.542089:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775768313.542091:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768313.542094:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4935 00000100:00080000:0.0:1775768313.542097:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768313.542100:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768313.542105:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768313.542125:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967410176/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768313.542216:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967409792:192.168.202.146@tcp:250:kworker.0 00000800:00000100:1.0:1775768313.542329:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768313.542370:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768313/real 1775768313] req@ffff8f9970a75c00 x1862022967409792/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768313.542388:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967409792/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768313.542397:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768313.542401:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768313.542405:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967409792:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775768313.542412:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967409920:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775768313.542433:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768313.542444:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768313/real 1775768313] req@ffff8f9970a77100 x1862022967409920/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768313.542461:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967409920/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768313.542469:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768313.542473:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768313.542476:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967409920:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775768313.542481:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967410048:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775768313.542499:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768313.542508:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768313/real 1775768313] req@ffff8f9970a76a00 x1862022967410048/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768313.542519:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967410048/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768313.542526:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768313.542529:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768313.542531:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967410048:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775768313.542539:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967410176:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775768313.542556:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768313.542566:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768313/real 1775768313] req@ffff8f9970a74380 x1862022967410176/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768313.542575:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967410176/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768313.542587:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768313.542590:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768313.542593:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967410176:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768314.437814:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768314.437857:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768315.461853:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768315.461914:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768316.485913:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768316.486234:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768317.509858:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768317.510014:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768318.533868:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768318.533940:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768318.661886:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768318.661895:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768318.661901:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768318.661911:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4940 00000100:00080000:3.0:1775768318.661924:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768318.661931:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768318.661970:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967410304/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768318.661999:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768318.662003:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768318.662004:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768318.662009:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4940 00000100:00080000:3.0:1775768318.662012:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768318.662017:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768318.662026:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768318.662033:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967410304:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775768318.662040:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967410432/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768318.662051:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768318.662054:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768318.662056:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768318.662060:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4940 00000100:00080000:3.0:1775768318.662064:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768318.662068:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768318.662072:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775768318.662085:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768318.662086:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967410560/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768318.662097:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775768318.662123:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768318/real 1775768318] req@ffff8f9970edd500 x1862022967410304/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768318.662147:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768318.662150:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775768318.662153:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967410304/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768318.662158:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4940 00000100:00080000:1.0:1775768318.662164:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768318.662165:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768318.662169:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00080000:3.0:1775768318.662172:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775768318.662174:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967410304:192.168.202.146@tcp:250:kworker.0 00000008:00100000:3.0:1775768318.662180:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768318.662193:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf100 x1862022967410688/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768318.662196:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967410432:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775768318.662217:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768318.662228:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768318/real 1775768318] req@ffff8f9970edce00 x1862022967410432/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768318.662238:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967410432/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768318.662247:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768318.662251:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768318.662254:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967410432:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775768318.662260:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967410560:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775768318.662280:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768318.662290:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768318/real 1775768318] req@ffff8f9970edc700 x1862022967410560/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768318.662298:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967410560/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768318.662303:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768318.662305:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768318.662306:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967410560:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775768318.662316:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967410688:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775768318.662328:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768318.662333:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768318/real 1775768318] req@ffff8f9970edf100 x1862022967410688/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768318.662343:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf100 x1862022967410688/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768318.662350:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768318.662353:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768318.662356:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967410688:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768319.557807:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768319.557919:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768320.581835:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768320.581883:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768321.605808:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768321.605864:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768322.629844:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768322.629899:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768323.653885:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768323.653910:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768323.781865:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768323.781874:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768323.781880:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768323.781888:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4945 00000100:00080000:3.0:1775768323.781899:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768323.781905:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768323.781939:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967410816/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768323.781966:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768323.781969:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768323.781974:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768323.781978:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4945 00000100:00080000:3.0:1775768323.781982:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768323.781986:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768323.781994:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768323.782008:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddc00 x1862022967410944/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768323.782011:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967410816:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775768323.782025:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768323.782028:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768323.782030:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768323.782034:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4945 00000100:00080000:3.0:1775768323.782038:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768323.782042:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768323.782048:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768323.782059:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967411072/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768323.782068:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775768323.782070:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768323.782074:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768323.782089:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768323.782093:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4945 00000100:00080000:3.0:1775768323.782097:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768323.782101:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768323.782106:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768323.782118:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967411200/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768323.782133:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768323/real 1775768323] req@ffff8f9970edc000 x1862022967410816/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768323.782147:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967410816/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768323.782154:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768323.782157:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768323.782160:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967410816:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775768323.782178:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967410944:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775768323.782192:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768323.782199:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768323/real 1775768323] req@ffff8f9970eddc00 x1862022967410944/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768323.782210:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddc00 x1862022967410944/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768323.782217:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768323.782220:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768323.782223:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967410944:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775768323.782229:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967411072:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775768323.782247:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768323.782256:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768323/real 1775768323] req@ffff8f9970edc380 x1862022967411072/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768323.782294:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967411072/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768323.782299:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768323.782301:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768323.782302:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967411072:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775768323.782306:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967411200:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775768323.782318:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768323.782324:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768323/real 1775768323] req@ffff8f9970edd880 x1862022967411200/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768323.782330:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967411200/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768323.782334:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768323.782336:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768323.782337:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967411200:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768324.677835:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768324.677879:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768325.701837:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768325.701906:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768326.725854:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768326.725919:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768327.749835:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768327.749887:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768328.773865:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768328.773928:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768328.901923:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768328.901940:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768328.901946:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768328.901954:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4950 00000100:00080000:0.0:1775768328.901968:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768328.901973:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775768328.902007:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967411328/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768328.902048:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768328.902052:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768328.902054:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768328.902095:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4950 00000100:00080000:0.0:1775768328.902098:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768328.902102:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768328.902109:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768328.902120:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967411456/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768328.902122:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967411328:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775768328.902146:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768328.902148:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768328.902149:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768328.902154:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4950 00000100:00080000:0.0:1775768328.902156:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768328.902158:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768328.902161:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768328.902168:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967411584/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768328.902174:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768328.902177:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775768328.902179:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768328.902181:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4950 00000100:00080000:0.0:1775768328.902199:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768328.902207:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:1.0:1775768328.902212:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:0.0:1775768328.902215:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768328.902227:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967411712/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768328.902278:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768328/real 1775768328] req@ffff8f9970a77800 x1862022967411328/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768328.902295:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967411328/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768328.902441:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768328.902448:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768328.902452:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967411328:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775768328.902490:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967411456:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775768328.902544:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768328.902568:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768328/real 1775768328] req@ffff8f9970a76d80 x1862022967411456/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768328.902584:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967411456/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768328.902592:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768328.902603:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768328.902607:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967411456:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775768328.902613:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967411584:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775768328.902634:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768328.902645:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768328/real 1775768328] req@ffff8f9970a74700 x1862022967411584/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768328.902657:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967411584/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768328.902665:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768328.902667:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768328.902671:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967411584:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775768328.902677:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967411712:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775768328.902696:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768328.902706:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768328/real 1775768328] req@ffff8f9970a74380 x1862022967411712/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768328.902717:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967411712/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768328.902725:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768328.902727:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768328.902730:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967411712:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768329.797841:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768329.797938:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768330.821888:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768330.821891:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768331.845808:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768331.845875:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768332.869881:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768332.869939:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768333.893862:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768333.893894:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775768334.022448:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768334.022462:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768334.022469:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768334.022484:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4955 00000100:00080000:2.0:1775768334.022498:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768334.022508:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775768334.022558:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3100 x1862022967411840/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768334.022595:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768334.022599:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768334.022601:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768334.022605:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4955 00000100:00080000:2.0:1775768334.022609:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768334.022613:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768334.022626:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768334.022640:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2300 x1862022967411968/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768334.022651:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768334.022655:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775768334.022656:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768334.022661:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4955 00000100:00080000:2.0:1775768334.022664:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768334.022668:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768334.022674:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768334.022681:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967411840:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775768334.022691:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1180 x1862022967412096/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768334.022704:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768334.022708:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775768334.022710:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768334.022715:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4955 00000100:00080000:2.0:1775768334.022719:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768334.022723:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768334.022729:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768334.022793:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3480 x1862022967412224/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768334.022817:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768334.022865:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768334/real 1775768334] req@ffff8f99703d3100 x1862022967411840/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768334.022884:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3100 x1862022967411840/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768334.022895:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768334.022899:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768334.022903:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967411840:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775768334.022911:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967411968:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775768334.022936:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768334.022948:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768334/real 1775768334] req@ffff8f99703d2300 x1862022967411968/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768334.022959:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2300 x1862022967411968/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768334.022966:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768334.022969:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768334.022972:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967411968:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775768334.023019:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967412096:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775768334.023041:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768334.023052:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768334/real 1775768334] req@ffff8f99703d1180 x1862022967412096/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768334.023062:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1180 x1862022967412096/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768334.023069:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768334.023072:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768334.023075:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967412096:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775768334.023081:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967412224:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775768334.023098:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768334.023108:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768334/real 1775768334] req@ffff8f99703d3480 x1862022967412224/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768334.023123:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3480 x1862022967412224/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768334.023137:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768334.023141:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768334.023144:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967412224:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768334.917856:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768334.917923:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768335.941868:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768335.941914:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775768336.965865:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768336.965941:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768337.989911:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768337.989928:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768339.013891:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768339.013978:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775768339.141861:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768339.141870:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768339.141876:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768339.141886:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4960 00000100:00080000:2.0:1775768339.141896:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768339.141902:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775768339.141946:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3b80 x1862022967412352/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768339.141981:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768339.141987:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768339.141990:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768339.141996:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4960 00000100:00080000:2.0:1775768339.142004:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768339.142009:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768339.142018:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768339.142033:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1880 x1862022967412480/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768339.142043:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967412352:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775768339.142049:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768339.142053:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775768339.142055:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768339.142059:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4960 00000100:00080000:2.0:1775768339.142062:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768339.142066:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768339.142071:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768339.142082:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2a00 x1862022967412608/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768339.142093:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768339.142097:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775768339.142100:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768339.142104:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4960 00000100:00080000:2.0:1775768339.142107:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768339.142112:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768339.142117:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775768339.142123:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768339.142134:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1c00 x1862022967412736/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768339.142195:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768339/real 1775768339] req@ffff8f99703d3b80 x1862022967412352/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768339.142225:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3b80 x1862022967412352/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768339.142236:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768339.142244:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768339.142251:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967412352:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775768339.142274:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967412480:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775768339.142299:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768339.142311:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768339/real 1775768339] req@ffff8f99703d1880 x1862022967412480/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768339.142324:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1880 x1862022967412480/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768339.142334:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768339.142337:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768339.142357:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967412480:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775768339.142363:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967412608:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768339.142384:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768339.142394:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768339/real 1775768339] req@ffff8f99703d2a00 x1862022967412608/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768339.142405:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2a00 x1862022967412608/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768339.142411:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768339.142414:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768339.142417:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967412608:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775768339.142422:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967412736:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768339.142439:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768339.142449:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768339/real 1775768339] req@ffff8f99703d1c00 x1862022967412736/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768339.142460:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1c00 x1862022967412736/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768339.142467:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768339.142469:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768339.142472:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967412736:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768340.037886:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768340.038001:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768341.061857:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768341.062076:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768342.085881:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768342.085941:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768343.109864:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768343.110773:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768344.133866:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768344.133939:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768344.262009:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768344.262019:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768344.262023:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768344.262031:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4965 00000100:00080000:1.0:1775768344.262042:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768344.262049:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775768344.262086:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967412864/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768344.262126:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768344.262130:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768344.262132:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768344.262136:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4965 00000100:00080000:1.0:1775768344.262141:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768344.262145:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768344.262154:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768344.262165:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967412992/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768344.262174:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768344.262177:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768344.262178:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768344.262181:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4965 00000100:00080000:1.0:1775768344.262184:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768344.262188:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768344.262192:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768344.262201:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967413120/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768344.262210:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768344.262213:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768344.262215:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768344.262235:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4965 00000100:00080000:1.0:1775768344.262241:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768344.262244:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768344.262249:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768344.262259:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967413248/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768344.262692:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967412864:192.168.202.146@tcp:250:kworker.0 00000800:00000100:0.0:1775768344.262770:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768344.262808:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768344/real 1775768344] req@ffff8f994f2e5180 x1862022967412864/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768344.262823:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967412864/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768344.262830:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768344.262832:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768344.262836:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967412864:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775768344.262840:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967412992:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775768344.262856:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768344.262864:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768344/real 1775768344] req@ffff8f994f2e4a80 x1862022967412992/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768344.262871:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967412992/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768344.262876:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768344.262891:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768344.262895:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967412992:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775768344.262901:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967413120:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768344.262915:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768344.262921:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768344/real 1775768344] req@ffff8f994f2e7800 x1862022967413120/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768344.262933:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967413120/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768344.262938:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768344.262940:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768344.262941:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967413120:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775768344.262945:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967413248:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768344.262954:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768344.262958:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768344/real 1775768344] req@ffff8f994f2e6d80 x1862022967413248/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768344.262964:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967413248/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768344.262969:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768344.262970:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768344.262972:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967413248:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768345.157912:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768345.157966:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768346.181989:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768346.184382:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775768347.205872:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768347.205950:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768348.229869:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768348.230015:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768349.253849:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768349.253911:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775768349.381894:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768349.381902:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768349.381906:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768349.381915:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4970 00000100:00080000:2.0:1775768349.381924:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768349.381929:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775768349.381964:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1c00 x1862022967413376/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768349.381998:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768349.382005:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768349.382006:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768349.382010:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4970 00000100:00080000:2.0:1775768349.382014:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768349.382018:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768349.382027:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768349.382040:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2a00 x1862022967413504/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768349.382047:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967413376:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775768349.382063:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768349.382066:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775768349.382068:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768349.382072:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4970 00000100:00080000:2.0:1775768349.382075:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768349.382079:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768349.382083:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768349.382093:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1880 x1862022967413632/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768349.382101:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768349.382104:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775768349.382106:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768349.382109:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4970 00000800:00000100:0.0:1775768349.382113:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775768349.382115:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768349.382120:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768349.382126:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768349.382136:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3b80 x1862022967413760/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768349.382170:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768349/real 1775768349] req@ffff8f99703d1c00 x1862022967413376/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768349.382198:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1c00 x1862022967413376/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768349.382246:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768349.382250:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768349.382254:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967413376:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775768349.382284:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967413504:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775768349.382308:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768349.382321:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768349/real 1775768349] req@ffff8f99703d2a00 x1862022967413504/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768349.382332:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2a00 x1862022967413504/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768349.382340:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768349.382343:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768349.382345:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967413504:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775768349.382350:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967413632:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768349.382369:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768349.382379:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768349/real 1775768349] req@ffff8f99703d1880 x1862022967413632/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768349.382390:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1880 x1862022967413632/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768349.382397:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768349.382400:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768349.382403:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967413632:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775768349.382409:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967413760:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768349.382428:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768349.382448:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768349/real 1775768349] req@ffff8f99703d3b80 x1862022967413760/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768349.382460:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3b80 x1862022967413760/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768349.382467:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768349.382470:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768349.382474:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967413760:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768350.277870:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768350.278074:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768351.301879:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768351.301966:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768352.325939:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768352.329121:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768353.349852:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768353.349933:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768354.373863:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768354.377973:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768354.504948:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768354.504957:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768354.504962:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768354.504972:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4976 00000100:00080000:1.0:1775768354.504983:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768354.504988:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775768354.505026:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6d80 x1862022967413888/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768354.505052:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768354.505055:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768354.505057:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768354.505061:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4976 00000100:00080000:1.0:1775768354.505064:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768354.505068:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768354.505077:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768354.505093:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967413888:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775768354.505093:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7800 x1862022967414016/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768354.505117:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768354.505121:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768354.505123:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1775768354.505163:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775768354.505167:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4976 00000100:00080000:1.0:1775768354.505172:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768354.505176:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768354.505182:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768354.505194:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4a80 x1862022967414144/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768354.505204:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768354.505207:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1775768354.505210:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768354/real 1775768354] req@ffff8f994f2e6d80 x1862022967413888/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768354.505210:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768354.505217:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4976 00000100:00080000:1.0:1775768354.505224:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768354.505229:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768354.505234:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768354.505237:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6d80 x1862022967413888/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768354.505249:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1775768354.505249:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5180 x1862022967414272/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768354.505257:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768354.505265:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967413888:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775768354.505289:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967414016:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775768354.505308:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768354.505315:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768354/real 1775768354] req@ffff8f994f2e7800 x1862022967414016/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768354.505323:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7800 x1862022967414016/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768354.505327:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768354.505330:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768354.505340:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967414016:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775768354.505345:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967414144:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768354.505363:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768354.505373:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768354/real 1775768354] req@ffff8f994f2e4a80 x1862022967414144/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768354.505383:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4a80 x1862022967414144/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768354.505391:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768354.505394:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768354.505397:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967414144:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775768354.505402:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967414272:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768354.505418:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768354.505434:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768354/real 1775768354] req@ffff8f994f2e5180 x1862022967414272/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768354.505465:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5180 x1862022967414272/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768354.505472:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768354.505475:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768354.505477:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967414272:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768355.397894:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768355.397946:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768356.422890:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768356.423570:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775768357.445871:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768357.446280:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768358.469870:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768358.470046:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768359.493928:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768359.493997:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768359.621887:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768359.621896:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768359.621900:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768359.621910:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4981 00000100:00080000:3.0:1775768359.621920:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768359.621926:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768359.621961:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967414400/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768359.621998:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768359.622002:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768359.622004:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768359.622028:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4981 00000100:00080000:3.0:1775768359.622032:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768359.622036:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768359.622046:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768359.622061:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967414400:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775768359.622065:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967414528/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768359.622076:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768359.622082:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768359.622084:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768359.622088:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4981 00000100:00080000:3.0:1775768359.622092:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768359.622096:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768359.622101:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768359.622111:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967414656/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768359.622120:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768359.622123:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768359.622125:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768359.622128:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4981 00000100:00080000:3.0:1775768359.622131:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768359.622135:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768359.622140:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768359.622163:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967414784/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768359.622183:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768359.622222:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768359/real 1775768359] req@ffff8f9970ede680 x1862022967414400/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768359.622236:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967414400/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768359.622244:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768359.622247:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768359.622250:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967414400:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775768359.622270:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967414528:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775768359.622285:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768359.622292:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768359/real 1775768359] req@ffff8f9970eded80 x1862022967414528/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768359.622299:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967414528/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768359.622317:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768359.622319:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768359.622321:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967414528:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775768359.622325:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967414656:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768359.622336:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768359.622342:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768359/real 1775768359] req@ffff8f9970edf480 x1862022967414656/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768359.622349:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967414656/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768359.622353:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768359.622355:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768359.622357:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967414656:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775768359.622360:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967414784:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768359.622368:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768359.622372:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768359/real 1775768359] req@ffff8f9970edf800 x1862022967414784/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768359.622378:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967414784/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768359.622383:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768359.622384:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768359.622395:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967414784:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768360.517900:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768360.519305:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768361.541869:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768361.542156:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768362.565852:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768362.565949:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768363.591040:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768363.591074:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775768364.613867:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768364.613923:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768364.741863:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768364.741907:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768364.741912:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768364.741922:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4986 00000100:00080000:3.0:1775768364.741932:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768364.741948:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768364.741990:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967414912/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768364.742016:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768364.742019:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768364.742021:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768364.742025:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4986 00000100:00080000:3.0:1775768364.742029:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768364.742033:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768364.742042:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768364.742060:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967414912:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775768364.742067:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967415040/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768364.742079:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768364.742082:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768364.742084:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768364.742088:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4986 00000100:00080000:3.0:1775768364.742092:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768364.742096:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768364.742100:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768364.742133:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967415168/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768364.742143:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768364.742146:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768364.742148:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1775768364.742152:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775768364.742153:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4986 00000100:00080000:3.0:1775768364.742159:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768364.742163:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768364.742168:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768364.742180:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967415296/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768364.742211:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768364/real 1775768364] req@ffff8f9970ede300 x1862022967414912/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768364.742228:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967414912/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768364.742237:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768364.742241:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768364.742245:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967414912:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775768364.742265:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967415040:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775768364.742284:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768364.742295:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768364/real 1775768364] req@ffff8f9970edfb80 x1862022967415040/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768364.742307:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967415040/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768364.742315:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768364.742319:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768364.742322:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967415040:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775768364.742327:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967415168:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768364.742363:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768364.742373:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768364/real 1775768364] req@ffff8f9970edca80 x1862022967415168/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768364.742385:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967415168/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768364.742392:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768364.742395:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768364.742398:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967415168:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775768364.742403:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967415296:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768364.742421:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768364.742431:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768364/real 1775768364] req@ffff8f9970edea00 x1862022967415296/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768364.742441:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967415296/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768364.742449:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768364.742451:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768364.742454:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967415296:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768365.637896:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768365.637967:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768366.661942:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768366.661948:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768367.685853:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768367.685965:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768368.709845:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768368.709914:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768369.733844:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768369.733903:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775768369.861855:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768369.861867:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768369.861871:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768369.861879:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4991 00000100:00080000:2.0:1775768369.861889:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768369.861894:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775768369.861923:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3480 x1862022967415424/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768369.861940:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768369.861942:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768369.861943:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768369.861946:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4991 00000100:00080000:2.0:1775768369.861948:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768369.861951:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768369.861958:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768369.861965:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1180 x1862022967415552/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768369.861971:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768369.861973:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775768369.861974:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768369.861976:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4991 00000100:00080000:2.0:1775768369.861978:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768369.861980:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768369.861983:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768369.861988:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967415424:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775768369.862024:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2300 x1862022967415680/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768369.862034:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768369.862037:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775768369.862038:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768369.862041:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4991 00000100:00080000:2.0:1775768369.862045:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775768369.862048:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775768369.862060:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768369.862065:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768369.862078:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3100 x1862022967415808/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768369.862102:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768369/real 1775768369] req@ffff8f99703d3480 x1862022967415424/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768369.862121:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3480 x1862022967415424/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768369.862131:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768369.862135:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768369.862138:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967415424:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775768369.862145:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967415552:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775768369.862164:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768369.862174:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768369/real 1775768369] req@ffff8f99703d1180 x1862022967415552/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768369.862185:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1180 x1862022967415552/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768369.862200:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768369.862202:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768369.862205:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967415552:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775768369.862227:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967415680:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768369.862243:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768369.862251:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768369/real 1775768369] req@ffff8f99703d2300 x1862022967415680/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768369.862261:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2300 x1862022967415680/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768369.862274:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768369.862277:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768369.862280:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967415680:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775768369.862284:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967415808:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768369.862296:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768369.862309:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768369/real 1775768369] req@ffff8f99703d3100 x1862022967415808/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768369.862316:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3100 x1862022967415808/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768369.862321:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768369.862322:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768369.862324:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967415808:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768370.757852:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768370.757911:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768371.781846:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768371.781907:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768372.806020:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768372.806114:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768373.829897:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768373.829962:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768374.853859:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768374.855770:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775768374.981833:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768374.981840:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768374.981843:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768374.981851:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 4996 00000100:00080000:2.0:1775768374.981861:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768374.981866:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775768374.981892:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2d80 x1862022967415936/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768374.981914:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768374.981917:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768374.981919:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768374.981922:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4996 00000100:00080000:2.0:1775768374.981924:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768374.981926:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768374.981932:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768374.981970:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1500 x1862022967416064/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768374.981976:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768374.981978:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775768374.981980:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768374.981983:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4996 00000100:00080000:2.0:1775768374.981986:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768374.981989:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768374.981994:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768374.982004:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0700 x1862022967416192/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768374.982008:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967415936:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775768374.982022:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768374.982025:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775768374.982027:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768374.982031:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 4996 00000100:00080000:2.0:1775768374.982035:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768374.982039:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768374.982044:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768374.982057:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0e00 x1862022967416320/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768374.982104:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967416064:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775768374.982137:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967416192:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775768374.982161:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967416320:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768375.877883:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768375.878186:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768376.901861:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768376.901926:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768377.925863:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768377.925888:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775768378.054842:0:2366:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:2.0:1775768378.054903:0:2366:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775768378.054919:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:2.0:1775768378.054961:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:2.0:1775768378.054970:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:2.0:1775768378.054977:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:0.0:1775768378.055042:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768374/real 1775768378] req@ffff8f99703d0700 x1862022967416192/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768378.055081:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0700 x1862022967416192/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768378.055093:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768378.055097:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768378.055101:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967416192:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775768378.055108:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768374/real 1775768378] req@ffff8f99703d0e00 x1862022967416320/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768378.055120:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0e00 x1862022967416320/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768378.055128:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768378.055131:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768378.055134:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967416320:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775768378.055141:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768374/real 1775768378] req@ffff8f99703d2d80 x1862022967415936/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768378.055151:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2d80 x1862022967415936/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768378.055158:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768378.055161:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768378.055164:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967415936:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775768378.055170:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768374/real 1775768378] req@ffff8f99703d1500 x1862022967416064/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768378.055193:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1500 x1862022967416064/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768378.055200:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768378.055204:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768378.055207:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967416064:192.168.202.146@tcp:38:kworker.0 00000100:00100000:2.0:1775768378.949879:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768378.949972:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768379.973807:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768379.974525:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768380.101884:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768380.101895:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768380.101900:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768380.101910:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 5001 00000100:00080000:3.0:1775768380.101921:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768380.101932:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768380.101988:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddf80 x1862022967416448/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768380.102020:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768380.102024:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768380.102026:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768380.102030:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5001 00000100:00080000:3.0:1775768380.102033:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768380.102037:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768380.102046:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768380.102059:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd180 x1862022967416576/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768380.102064:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967416448:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775768380.102082:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768380.102086:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768380.102087:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768380.102092:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5001 00000100:00080000:3.0:1775768380.102096:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768380.102100:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768380.102105:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768380.102117:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edea00 x1862022967416704/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768380.102127:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775768380.102127:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768380.102132:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768380.102134:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768380.102139:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5001 00000100:00080000:3.0:1775768380.102144:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768380.102147:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768380.102152:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768380.102162:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edca80 x1862022967416832/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768380.102176:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768380/real 1775768380] req@ffff8f9970eddf80 x1862022967416448/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768380.102197:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddf80 x1862022967416448/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768380.102207:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768380.102211:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768380.102215:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967416448:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775768380.102241:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967416576:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775768380.102267:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768380.102279:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768380/real 1775768380] req@ffff8f9970edd180 x1862022967416576/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768380.102291:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd180 x1862022967416576/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768380.102304:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768380.102307:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768380.102311:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967416576:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775768380.102322:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967416704:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768380.102341:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768380.102351:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768380/real 1775768380] req@ffff8f9970edea00 x1862022967416704/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768380.102361:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edea00 x1862022967416704/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768380.102369:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768380.102373:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768380.102375:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967416704:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775768380.102381:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967416832:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768380.102405:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768380.102414:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768380/real 1775768380] req@ffff8f9970edca80 x1862022967416832/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768380.102423:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edca80 x1862022967416832/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768380.102429:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768380.102434:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768380.102457:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967416832:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768380.997894:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768380.998187:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768382.021843:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768382.022160:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768383.045852:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768383.045921:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768384.069858:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768384.070040:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768385.093962:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768385.094452:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775768385.221863:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768385.221872:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768385.221877:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768385.221886:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 5006 00000100:00080000:2.0:1775768385.221897:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768385.221903:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775768385.221936:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3800 x1862022967416960/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768385.222015:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967416960:192.168.202.146@tcp:250:kworker.0 00000800:00000100:0.0:1775768385.222068:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775768385.222097:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768385.222101:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768385.222103:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768385.222107:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5006 00000100:00080000:2.0:1775768385.222111:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768385.222115:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768385.222123:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768385.222136:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0380 x1862022967417088/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768385.222196:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768385/real 1775768385] req@ffff8f99703d3800 x1862022967416960/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768385.222213:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3800 x1862022967416960/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768385.222223:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768385.222233:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768385.222240:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967416960:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775768385.222260:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967417088:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775768385.222278:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768385.222288:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768385/real 1775768385] req@ffff8f99703d0380 x1862022967417088/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768385.222299:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0380 x1862022967417088/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768385.222306:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768385.222309:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768385.222312:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967417088:192.168.202.146@tcp:38:kworker.0 00000100:00080000:2.0:1775768385.222391:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768385.222395:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775768385.222397:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768385.222402:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5006 00000100:00080000:2.0:1775768385.222405:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768385.222409:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768385.222414:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768385.222425:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0000 x1862022967417216/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768385.222440:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768385.222443:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775768385.222448:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768385.222451:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5006 00000100:00080000:2.0:1775768385.222454:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768385.222458:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768385.222462:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768385.222472:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2680 x1862022967417344/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768385.222474:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967417216:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768385.222519:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768385.222534:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768385/real 1775768385] req@ffff8f99703d0000 x1862022967417216/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768385.222547:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0000 x1862022967417216/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768385.222555:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768385.222558:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768385.222562:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967417216:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775768385.222575:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967417344:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768385.222593:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768385.222604:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768385/real 1775768385] req@ffff8f99703d2680 x1862022967417344/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768385.222615:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2680 x1862022967417344/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768385.222623:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768385.222626:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768385.222629:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967417344:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768386.117860:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768386.118149:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768387.141865:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768387.142010:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768388.165935:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768388.165998:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768389.189874:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768389.189923:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768390.213830:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768390.213887:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768390.341909:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768390.341920:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768390.341924:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768390.341934:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 5011 00000100:00080000:1.0:1775768390.341944:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768390.341950:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775768390.342030:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6a00 x1862022967417472/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768390.342067:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768390.342074:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768390.342076:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768390.342080:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5011 00000100:00080000:1.0:1775768390.342084:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768390.342087:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768390.342100:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768390.342114:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e5f80 x1862022967417600/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768390.342142:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768390.342145:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768390.342147:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768390.342152:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5011 00000100:00080000:1.0:1775768390.342155:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768390.342161:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775768390.342162:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967417472:192.168.202.146@tcp:250:kworker.0 00000008:00100000:1.0:1775768390.342166:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768390.342198:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4000 x1862022967417728/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768390.342209:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768390.342212:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768390.342214:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768390.342217:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5011 00000100:00080000:1.0:1775768390.342221:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775768390.342224:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775768390.342228:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768390.342234:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768390.342261:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6680 x1862022967417856/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768390.342267:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768390/real 1775768390] req@ffff8f994f2e6a00 x1862022967417472/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768390.342297:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6a00 x1862022967417472/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768390.342307:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768390.342312:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768390.342316:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967417472:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775768390.342323:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967417600:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775768390.342345:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768390.342357:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768390/real 1775768390] req@ffff8f994f2e5f80 x1862022967417600/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768390.342369:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e5f80 x1862022967417600/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768390.342377:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768390.342380:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768390.342383:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967417600:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775768390.342406:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967417728:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768390.342426:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768390.342455:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768390/real 1775768390] req@ffff8f994f2e4000 x1862022967417728/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768390.342467:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4000 x1862022967417728/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768390.342498:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768390.342501:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768390.342504:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967417728:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775768390.342511:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967417856:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768390.342531:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768390.342542:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768390/real 1775768390] req@ffff8f994f2e6680 x1862022967417856/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768390.342552:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6680 x1862022967417856/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768390.342560:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768390.342563:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768390.342566:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967417856:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768391.237823:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768391.237892:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768392.261857:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768392.261943:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768393.285846:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768393.285923:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768394.309994:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768394.310218:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768395.333930:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768395.334001:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00080000:1.0:1775768395.461854:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768395.461950:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768395.461955:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768395.461964:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 5017 00000100:00080000:1.0:1775768395.461975:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768395.461981:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775768395.462032:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967417984/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768395.462061:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768395.462065:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768395.462067:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768395.462083:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5017 00000100:00080000:1.0:1775768395.462087:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768395.462090:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768395.462099:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768395.462122:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967418112/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768395.462131:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768395.462134:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768395.462136:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768395.462139:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5017 00000100:00080000:1.0:1775768395.462142:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768395.462146:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768395.462151:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768395.462172:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967418240/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768395.462190:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768395.462193:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768395.462194:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768395.462198:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5017 00000100:00080000:1.0:1775768395.462201:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768395.462204:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768395.462208:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768395.462217:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967418368/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768395.462311:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967417984:192.168.202.146@tcp:250:kworker.0 00000800:00000100:0.0:1775768395.462367:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768395.462418:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768395/real 1775768395] req@ffff8f994f2e6300 x1862022967417984/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768395.462484:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967417984/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768395.462494:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768395.462503:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768395.462521:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967417984:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775768395.462528:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967418112:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775768395.462550:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768395.462561:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768395/real 1775768395] req@ffff8f994f2e4e00 x1862022967418112/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768395.462573:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967418112/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768395.462580:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768395.462583:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768395.462586:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967418112:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775768395.462592:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967418240:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768395.462609:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768395.462833:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768395/real 1775768395] req@ffff8f994f2e7b80 x1862022967418240/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768395.462845:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967418240/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768395.462852:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768395.462864:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768395.462867:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967418240:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775768395.462873:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967418368:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768395.462893:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768395.462917:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768395/real 1775768395] req@ffff8f994f2e4700 x1862022967418368/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768395.462928:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967418368/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768395.462936:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768395.462939:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768395.462942:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967418368:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768396.357865:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768396.357919:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768397.381948:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768397.381962:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768398.405980:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768398.406019:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775768399.429850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768399.429911:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768400.453857:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768400.453907:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768400.582617:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768400.582628:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768400.582634:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768400.582644:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 5022 00000100:00080000:3.0:1775768400.582656:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768400.582664:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768400.582701:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edfb80 x1862022967418496/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768400.582732:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768400.582777:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768400.582779:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768400.582784:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5022 00000100:00080000:3.0:1775768400.582789:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768400.582793:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768400.582803:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768400.582816:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede300 x1862022967418624/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768400.582827:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768400.582830:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768400.582832:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768400.582836:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5022 00000100:00080000:3.0:1775768400.582840:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768400.582843:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768400.582848:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768400.582860:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967418752/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768400.582870:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768400.582874:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768400.582875:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768400.582878:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5022 00000100:00080000:3.0:1775768400.582882:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775768400.583003:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967418496:192.168.202.146@tcp:250:kworker.0 00000800:00000100:0.0:1775768400.583064:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768400.583212:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768400/real 1775768400] req@ffff8f9970edfb80 x1862022967418496/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768400.583231:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edfb80 x1862022967418496/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768400.583240:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768400.583244:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768400.583248:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967418496:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775768400.583255:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967418624:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775768400.583276:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768400.583289:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768400/real 1775768400] req@ffff8f9970ede300 x1862022967418624/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768400.583301:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede300 x1862022967418624/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768400.583350:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768400.583354:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768400.583358:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967418624:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775768400.583365:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967418752:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768400.583386:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768400.583396:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768400/real 1775768400] req@ffff8f9970edf800 x1862022967418752/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768400.583408:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967418752/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768400.583416:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768400.583419:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768400.583422:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967418752:192.168.202.146@tcp:8:kworker.0 00000100:00080000:3.0:1775768400.583516:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768400.583525:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768400.583540:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967418880/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768400.584633:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967418880:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768400.584673:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768400.584690:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768400/real 1775768400] req@ffff8f9970edf480 x1862022967418880/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768400.584702:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967418880/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768400.584709:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768400.584713:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768400.584773:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967418880:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768401.477849:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768401.477938:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768402.501875:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768402.501889:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775768403.525854:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768403.525975:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768404.549888:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768404.549998:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768405.573865:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768405.573996:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775768405.701878:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768405.701887:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768405.701891:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768405.701900:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 5027 00000100:00080000:2.0:1775768405.701910:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768405.701947:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775768405.702001:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2680 x1862022967419008/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768405.702026:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768405.702029:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768405.702034:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768405.702038:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5027 00000100:00080000:2.0:1775768405.702041:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768405.702044:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768405.702051:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768405.702067:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967419008:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775768405.702078:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0000 x1862022967419136/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768405.702099:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768405.702102:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775768405.702105:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768405.702109:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5027 00000100:00080000:2.0:1775768405.702113:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768405.702116:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768405.702122:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768405.702134:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0380 x1862022967419264/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768405.702154:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775768405.702155:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768405.702158:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775768405.702160:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768405.702164:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5027 00000100:00080000:2.0:1775768405.702169:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768405.702173:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768405.702178:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768405.702190:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3800 x1862022967419392/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768405.702229:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768405/real 1775768405] req@ffff8f99703d2680 x1862022967419008/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768405.702257:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2680 x1862022967419008/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768405.702267:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768405.702271:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768405.702275:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967419008:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775768405.702322:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967419136:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775768405.702341:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768405.702350:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768405/real 1775768405] req@ffff8f99703d0000 x1862022967419136/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768405.702358:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0000 x1862022967419136/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768405.702363:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768405.702365:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768405.702367:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967419136:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775768405.702371:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967419264:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768405.702383:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768405.702393:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768405/real 1775768405] req@ffff8f99703d0380 x1862022967419264/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768405.702400:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0380 x1862022967419264/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768405.702404:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768405.702406:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768405.702407:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967419264:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775768405.702410:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967419392:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768405.702420:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768405.702427:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768405/real 1775768405] req@ffff8f99703d3800 x1862022967419392/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768405.702455:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3800 x1862022967419392/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768405.702463:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768405.702466:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768405.702469:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967419392:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768406.597892:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768406.598188:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768407.621851:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768407.621904:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768408.645852:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768408.646428:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768409.669860:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768409.669880:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775768410.693843:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768410.693907:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775768410.821856:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768410.821865:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768410.821870:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768410.821880:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 5032 00000100:00080000:2.0:1775768410.821890:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768410.821896:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775768410.821935:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1500 x1862022967419520/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768410.821961:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768410.821965:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768410.821967:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768410.821971:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5032 00000100:00080000:2.0:1775768410.821974:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768410.821977:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768410.821990:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768410.822003:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2d80 x1862022967419648/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768410.822013:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768410.822017:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1775768410.822020:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967419520:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775768410.822020:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768410.822027:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5032 00000100:00080000:2.0:1775768410.822031:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768410.822035:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768410.822040:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768410.822059:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0e00 x1862022967419776/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768410.822068:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768410.822071:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775768410.822074:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768410.822078:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5032 00000100:00080000:2.0:1775768410.822082:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775768410.822087:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775768410.822087:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768410.822095:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768410.822106:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d0700 x1862022967419904/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768410.822130:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768410/real 1775768410] req@ffff8f99703d1500 x1862022967419520/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768410.822158:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1500 x1862022967419520/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768410.822169:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768410.822173:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768410.822176:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967419520:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775768410.822200:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967419648:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775768410.822226:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768410.822253:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768410/real 1775768410] req@ffff8f99703d2d80 x1862022967419648/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768410.822267:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2d80 x1862022967419648/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768410.822276:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768410.822280:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768410.822283:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967419648:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775768410.822290:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967419776:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768410.822311:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768410.822322:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768410/real 1775768410] req@ffff8f99703d0e00 x1862022967419776/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768410.822334:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0e00 x1862022967419776/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768410.822348:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768410.822351:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768410.822355:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967419776:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775768410.822361:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967419904:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768410.822380:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768410.822389:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768410/real 1775768410] req@ffff8f99703d0700 x1862022967419904/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768410.822402:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d0700 x1862022967419904/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768410.822410:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768410.822413:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768410.822416:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967419904:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768411.717817:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768411.717923:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768412.741811:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768412.741869:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768413.765843:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768413.765955:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768414.789841:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768414.789897:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768415.813847:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768415.813912:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775768415.941864:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768415.941872:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768415.941876:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768415.941886:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 5037 00000100:00080000:2.0:1775768415.941896:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768415.941902:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775768415.941936:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3100 x1862022967420032/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768415.941964:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768415.941968:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768415.941969:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768415.941973:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5037 00000100:00080000:2.0:1775768415.941976:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768415.941979:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768415.941987:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768415.941997:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2300 x1862022967420160/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768415.942008:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768415.942010:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775768415.942012:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775768415.942017:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967420032:192.168.202.146@tcp:250:kworker.0 00000100:00080000:2.0:1775768415.942018:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5037 00000100:00080000:2.0:1775768415.942023:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768415.942028:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768415.942032:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768415.942041:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1180 x1862022967420288/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768415.942049:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768415.942052:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775768415.942053:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768415.942056:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5037 00000100:00080000:2.0:1775768415.942060:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768415.942063:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768415.942068:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775768415.942087:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:2.0:1775768415.942112:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3480 x1862022967420416/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768415.942129:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768415/real 1775768415] req@ffff8f99703d3100 x1862022967420032/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768415.942148:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3100 x1862022967420032/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768415.942157:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768415.942161:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768415.942183:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967420032:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775768415.942242:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967420160:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775768415.942264:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768415.942276:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768415/real 1775768415] req@ffff8f99703d2300 x1862022967420160/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768415.942288:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2300 x1862022967420160/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768415.942295:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768415.942298:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768415.942301:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967420160:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775768415.942320:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967420288:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768415.942339:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768415.942348:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768415/real 1775768415] req@ffff8f99703d1180 x1862022967420288/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768415.942358:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1180 x1862022967420288/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768415.942365:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768415.942368:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768415.942371:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967420288:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775768415.942376:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967420416:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768415.942391:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768415.942401:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768415/real 1775768415] req@ffff8f99703d3480 x1862022967420416/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768415.942420:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3480 x1862022967420416/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768415.942427:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768415.942430:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768415.942433:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967420416:192.168.202.146@tcp:8:kworker.0 00010000:00010000:0.0:1775768416.837929:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768416.837983:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768417.861909:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768417.861956:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775768418.885841:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768418.885890:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768419.909826:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768419.909888:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768420.933844:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768420.934046:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768421.061887:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768421.061939:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768421.061943:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768421.061952:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 5042 00000100:00080000:3.0:1775768421.061974:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768421.061980:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768421.062033:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967420544/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768421.062058:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768421.062061:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768421.062063:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768421.062067:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5042 00000100:00080000:3.0:1775768421.062070:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768421.062074:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768421.062082:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768421.062094:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967420544:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775768421.062095:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967420672/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768421.062115:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768421.062118:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768421.062123:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768421.062127:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5042 00000100:00080000:3.0:1775768421.062132:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768421.062134:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768421.062150:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775768421.062166:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768421.062170:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967420800/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768421.062178:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768421.062186:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768421.062187:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768421.062190:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5042 00000100:00080000:3.0:1775768421.062194:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768421.062196:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768421.062200:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768421.062212:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967420928/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768421.062216:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768421/real 1775768421] req@ffff8f9970eded80 x1862022967420544/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768421.062240:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967420544/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768421.062251:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768421.062263:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768421.062267:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967420544:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775768421.062290:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967420672:192.168.202.146@tcp:38:kworker.0 00000800:00000100:0.0:1775768421.062307:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768421.062315:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768421/real 1775768421] req@ffff8f9970ede680 x1862022967420672/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768421.062322:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967420672/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768421.062341:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768421.062344:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768421.062346:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967420672:192.168.202.146@tcp:38:kworker.0 00000100:00100000:0.0:1775768421.062350:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967420800:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768421.062361:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768421.062367:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768421/real 1775768421] req@ffff8f9970edd880 x1862022967420800/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768421.062374:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967420800/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768421.062379:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768421.062381:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768421.062382:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967420800:192.168.202.146@tcp:8:kworker.0 00000100:00100000:0.0:1775768421.062385:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967420928:192.168.202.146@tcp:8:kworker.0 00000800:00000100:0.0:1775768421.062393:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:0.0:1775768421.062398:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768421/real 1775768421] req@ffff8f9970edc380 x1862022967420928/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768421.062404:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967420928/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768421.062411:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768421.062413:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:0.0:1775768421.062417:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967420928:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768421.957864:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768421.957945:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768422.981853:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768422.981956:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768424.005909:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768424.005916:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768425.029826:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768425.029894:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768426.053864:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768426.053927:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768426.181865:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768426.181874:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768426.181879:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768426.181889:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 5047 00000100:00080000:0.0:1775768426.181900:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768426.181906:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775768426.181978:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967421056/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768426.182006:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768426.182010:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768426.182015:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768426.182021:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5047 00000100:00080000:0.0:1775768426.182024:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768426.182028:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768426.182036:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768426.182058:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967421056:192.168.202.146@tcp:250:kworker.0 00000100:00100000:0.0:1775768426.182063:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967421184/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768426.182074:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768426.182077:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768426.182078:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768426.182082:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5047 00000100:00080000:0.0:1775768426.182098:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768426.182114:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768426.182119:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768426.182140:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967421312/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768426.182148:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768426.182151:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775768426.182153:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768426.182156:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5047 00000100:00080000:0.0:1775768426.182159:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768426.182162:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768426.182166:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768426.182175:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967421440/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768426.182252:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768426.182298:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768426/real 1775768426] req@ffff8f9970a76a00 x1862022967421056/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768426.182317:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967421056/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768426.182326:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768426.182330:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768426.182334:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967421056:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775768426.182355:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967421184:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775768426.182376:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768426.182393:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768426/real 1775768426] req@ffff8f9970a77100 x1862022967421184/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768426.182404:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967421184/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768426.182411:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768426.182414:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768426.182416:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967421184:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775768426.182436:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967421312:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775768426.182455:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768426.182465:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768426/real 1775768426] req@ffff8f9970a75c00 x1862022967421312/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768426.182476:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967421312/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768426.182483:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768426.182486:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768426.182489:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967421312:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775768426.182494:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967421440:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775768426.182511:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768426.182520:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768426/real 1775768426] req@ffff8f9970a75500 x1862022967421440/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768426.182528:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967421440/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768426.182533:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768426.182535:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768426.182537:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967421440:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768427.077854:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768427.077948:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768428.101882:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768428.101932:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768429.125851:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768429.125901:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768430.149838:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768430.149900:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768431.173899:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768431.173899:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00080000:2.0:1775768431.301889:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768431.301899:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768431.301910:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768431.301920:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 5052 00000100:00080000:2.0:1775768431.301932:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768431.301938:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:2.0:1775768431.301975:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d3b80 x1862022967421568/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768431.302001:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768431.302006:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768431.302008:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768431.302012:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5052 00000100:00080000:2.0:1775768431.302016:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768431.302019:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768431.302028:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768431.302069:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967421568:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775768431.302070:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1880 x1862022967421696/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768431.302085:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768431.302088:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775768431.302090:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768431.302094:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5052 00000100:00080000:2.0:1775768431.302098:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768431.302102:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768431.302107:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768431.302118:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d2a00 x1862022967421824/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768431.302127:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768431.302130:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775768431.302132:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1775768431.302137:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:2.0:1775768431.302141:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5052 00000100:00080000:2.0:1775768431.302145:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768431.302148:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:2.0:1775768431.302154:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768431.302166:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f99703d1c00 x1862022967421952/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768431.302179:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768431/real 1775768431] req@ffff8f99703d3b80 x1862022967421568/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768431.302198:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d3b80 x1862022967421568/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768431.302208:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768431.302211:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768431.302215:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967421568:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775768431.302249:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967421696:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775768431.302271:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768431.302282:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768431/real 1775768431] req@ffff8f99703d1880 x1862022967421696/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768431.302294:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1880 x1862022967421696/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768431.302302:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768431.302306:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768431.302309:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967421696:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775768431.302314:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967421824:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775768431.302334:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768431.302345:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768431/real 1775768431] req@ffff8f99703d2a00 x1862022967421824/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768431.302361:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d2a00 x1862022967421824/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768431.302368:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768431.302371:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768431.302374:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967421824:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775768431.302379:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967421952:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775768431.302399:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768431.302409:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768431/real 1775768431] req@ffff8f99703d1c00 x1862022967421952/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768431.302419:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f99703d1c00 x1862022967421952/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768431.302441:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768431.302445:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768431.302449:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f99703d1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967421952:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768432.197834:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768432.197916:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768433.221846:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768433.221903:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768434.245862:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768434.246066:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768435.269850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768435.269996:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768436.293877:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768436.293951:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768436.421909:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768436.421916:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768436.421920:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768436.421933:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 5058 00000100:00080000:3.0:1775768436.421942:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768436.421946:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768436.421976:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddc00 x1862022967422080/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768436.422013:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768436.422016:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768436.422017:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768436.422021:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5058 00000100:00080000:3.0:1775768436.422023:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768436.422026:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768436.422034:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768436.422042:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967422208/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768436.422057:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967422080:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775768436.422082:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768436.422084:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768436.422086:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768436.422089:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5058 00000100:00080000:3.0:1775768436.422091:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768436.422094:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768436.422098:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768436.422110:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf100 x1862022967422336/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775768436.422115:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775768436.422129:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768436.422132:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768436.422134:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768436.422139:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5058 00000100:00080000:3.0:1775768436.422142:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768436.422145:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768436.422149:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768436.422156:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967422464/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768436.422164:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768436/real 1775768436] req@ffff8f9970eddc00 x1862022967422080/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768436.422198:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddc00 x1862022967422080/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768436.422208:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768436.422212:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768436.422215:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967422080:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775768436.422236:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967422208:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775768436.422255:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768436.422271:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768436/real 1775768436] req@ffff8f9970edc000 x1862022967422208/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768436.422279:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967422208/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768436.422284:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768436.422286:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768436.422288:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967422208:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775768436.422292:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967422336:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775768436.422302:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768436.422315:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768436/real 1775768436] req@ffff8f9970edf100 x1862022967422336/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768436.422322:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf100 x1862022967422336/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768436.422343:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768436.422345:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768436.422348:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967422336:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775768436.422353:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967422464:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775768436.422369:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768436.422376:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768436/real 1775768436] req@ffff8f9970edc700 x1862022967422464/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768436.422382:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967422464/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768436.422387:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768436.422389:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768436.422391:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967422464:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768437.317846:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768437.317909:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768438.341909:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768438.341920:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768439.365910:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768439.366004:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768440.389850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768440.389907:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768441.413850:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768441.413986:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768441.541850:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768441.541858:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768441.541863:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768441.541872:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 5063 00000100:00080000:3.0:1775768441.541924:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768441.541929:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768441.541977:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edce00 x1862022967422592/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768441.542004:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768441.542006:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768441.542007:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768441.542011:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5063 00000100:00080000:3.0:1775768441.542014:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768441.542016:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768441.542024:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768441.542036:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd500 x1862022967422720/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768441.542046:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768441.542051:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1775768441.542052:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967422592:192.168.202.146@tcp:250:kworker.0 00000100:00080000:3.0:1775768441.542053:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768441.542061:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5063 00000100:00080000:3.0:1775768441.542065:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768441.542069:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768441.542074:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768441.542086:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc700 x1862022967422848/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768441.542095:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768441.542109:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768441.542110:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768441.542113:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5063 00000100:00080000:3.0:1775768441.542116:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768441.542118:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768441.542122:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768441.542129:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf100 x1862022967422976/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768441.542161:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967422720:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775768441.542187:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967422848:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775768441.542305:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967422976:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768442.437835:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768442.437898:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768443.461846:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768443.461905:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768444.485972:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768444.487066:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000400:00000100:2.0:1775768444.614073:0:2368:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.146:988 00000400:00000100:2.0:1775768444.614114:0:2368:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.146@tcp at host 192.168.202.146 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775768444.614134:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:2.0:1775768444.614179:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:2.0:1775768444.614189:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000800:00000100:2.0:1775768444.614197:0:2368:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.146@tcp 00000100:00100000:1.0:1775768444.614224:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768441/real 1775768444] req@ffff8f9970edd500 x1862022967422720/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768444.614248:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd500 x1862022967422720/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768444.614258:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768444.614263:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768444.614267:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967422720:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775768444.614274:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768441/real 1775768444] req@ffff8f9970edc700 x1862022967422848/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768444.614285:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc700 x1862022967422848/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768444.614292:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768444.614296:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768444.614299:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967422848:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775768444.614305:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768441/real 1775768444] req@ffff8f9970edf100 x1862022967422976/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768444.614315:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf100 x1862022967422976/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768444.614322:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768444.614326:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768444.614328:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967422976:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775768444.614334:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768441/real 1775768444] req@ffff8f9970edce00 x1862022967422592/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768444.614345:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edce00 x1862022967422592/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768444.614352:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768444.614355:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768444.614358:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967422592:192.168.202.146@tcp:250:kworker.0 00000100:00100000:2.0:1775768445.509929:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768445.509942:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768446.533859:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768446.533936:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768446.662082:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768446.662087:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768446.662090:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768446.662098:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 5068 00000100:00080000:0.0:1775768446.662106:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768446.662112:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775768446.662142:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77480 x1862022967423104/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768446.662171:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768446.662175:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768446.662177:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768446.662182:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5068 00000100:00080000:0.0:1775768446.662186:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768446.662189:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768446.662195:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967423104:192.168.202.146@tcp:250:kworker.0 00000008:00100000:0.0:1775768446.662198:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768446.662214:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75180 x1862022967423232/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768446.662224:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768446.662227:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768446.662228:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768446.662232:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5068 00000100:00080000:0.0:1775768446.662235:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768446.662239:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768446.662243:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768446.662255:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76300 x1862022967423360/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768446.662265:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768446.662268:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775768446.662269:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768446.662273:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5068 00000100:00080000:0.0:1775768446.662278:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768446.662281:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768446.662286:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768446.662296:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75500 x1862022967423488/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775768446.662309:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768446.662352:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768446/real 1775768446] req@ffff8f9970a77480 x1862022967423104/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768446.662370:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77480 x1862022967423104/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768446.662380:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768446.662384:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768446.662388:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967423104:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775768446.662412:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967423232:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775768446.662434:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768446.662542:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768446/real 1775768446] req@ffff8f9970a75180 x1862022967423232/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768446.662556:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75180 x1862022967423232/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768446.662586:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768446.662590:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768446.662593:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967423232:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775768446.662599:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967423360:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768446.662626:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768446.662635:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768446/real 1775768446] req@ffff8f9970a76300 x1862022967423360/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768446.662645:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76300 x1862022967423360/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768446.662778:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768446.662782:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768446.662785:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967423360:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775768446.662790:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967423488:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768446.662850:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768446.662863:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768446/real 1775768446] req@ffff8f9970a75500 x1862022967423488/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768446.662873:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75500 x1862022967423488/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768446.662880:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768446.662884:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768446.662908:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967423488:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768447.557842:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768447.557909:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768448.581858:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768448.581982:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768449.605871:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768449.608931:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768450.629847:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768450.629909:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768451.653863:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768451.654984:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768451.784923:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768451.784933:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768451.784937:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768451.784948:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 5073 00000100:00080000:3.0:1775768451.784958:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768451.784965:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768451.785001:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc000 x1862022967423616/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768451.785102:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967423616:192.168.202.146@tcp:250:kworker.0 00000800:00000100:1.0:1775768451.785251:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775768451.785285:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768451.785290:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768451.785292:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768451.785297:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5073 00000100:00080000:3.0:1775768451.785302:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768451.785306:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768451.785315:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768451.785329:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eddc00 x1862022967423744/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768451.785390:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768451/real 1775768451] req@ffff8f9970edc000 x1862022967423616/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768451.785408:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc000 x1862022967423616/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768451.785418:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768451.785421:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768451.785425:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967423616:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775768451.785449:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967423744:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775768451.785472:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768451.785484:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768451/real 1775768451] req@ffff8f9970eddc00 x1862022967423744/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768451.785495:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eddc00 x1862022967423744/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768451.785503:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768451.785506:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768451.785509:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967423744:192.168.202.146@tcp:38:kworker.0 00000100:00080000:3.0:1775768451.785611:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768451.785615:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768451.785617:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768451.785622:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5073 00000100:00080000:3.0:1775768451.785626:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768451.785646:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768451.785651:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768451.785664:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edc380 x1862022967423872/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768451.785708:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967423872:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775768451.785731:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775768451.785772:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768451.785776:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768451.785778:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768451.785781:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5073 00000100:00080000:3.0:1775768451.785786:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768451.785789:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768451.785794:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768451.785805:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edd880 x1862022967424000/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768451.785911:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768451/real 1775768451] req@ffff8f9970edc380 x1862022967423872/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768451.785923:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edc380 x1862022967423872/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768451.785930:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768451.785934:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768451.785937:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967423872:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775768451.785947:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967424000:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775768451.785966:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768451.785976:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768451/real 1775768451] req@ffff8f9970edd880 x1862022967424000/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768451.785987:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edd880 x1862022967424000/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768451.785994:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768451.785997:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768451.785999:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967424000:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768452.677918:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768452.677937:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768453.701864:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768453.701930:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768454.725845:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768454.725926:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768455.749874:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768455.749927:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768456.773902:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768456.776864:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768456.901970:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768456.901985:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768456.901990:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768456.901999:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 5078 00000100:00080000:0.0:1775768456.902010:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768456.902016:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775768456.902052:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a75c00 x1862022967424128/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768456.902080:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768456.902083:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768456.902085:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768456.902089:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5078 00000100:00080000:0.0:1775768456.902093:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768456.902096:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768456.902105:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768456.902122:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77100 x1862022967424256/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768456.902128:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967424128:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775768456.902144:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768456.902146:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768456.902148:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768456.902153:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5078 00000100:00080000:0.0:1775768456.902156:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768456.902160:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768456.902165:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768456.902174:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76a00 x1862022967424384/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768456.902183:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768456.902186:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775768456.902188:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1775768456.902193:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:0.0:1775768456.902196:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5078 00000100:00080000:0.0:1775768456.902201:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768456.902205:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768456.902210:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768456.902239:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768456/real 1775768456] req@ffff8f9970a75c00 x1862022967424128/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768456.902258:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967424512/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768456.902259:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a75c00 x1862022967424128/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768456.902281:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768456.902285:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768456.902289:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a75c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967424128:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775768456.902311:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967424256:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775768456.902331:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768456.902341:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768456/real 1775768456] req@ffff8f9970a77100 x1862022967424256/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768456.902373:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77100 x1862022967424256/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768456.902380:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768456.902383:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768456.902386:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967424256:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775768456.902391:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967424384:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775768456.902406:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768456.902413:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768456/real 1775768456] req@ffff8f9970a76a00 x1862022967424384/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768456.902423:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76a00 x1862022967424384/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768456.902429:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768456.902431:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768456.902434:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967424384:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775768456.902438:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967424512:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775768456.902450:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768456.902466:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768456/real 1775768456] req@ffff8f9970a74380 x1862022967424512/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768456.902477:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967424512/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768456.902490:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768456.902493:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768456.902496:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967424512:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768457.797882:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768457.797953:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768458.821868:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768458.821931:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768459.845918:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768459.846807:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768460.869894:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768460.869954:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768461.893910:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768461.893927:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768462.022137:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768462.022146:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768462.022150:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768462.022159:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 5083 00000100:00080000:1.0:1775768462.022169:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768462.022174:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:1.0:1775768462.022212:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4700 x1862022967424640/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768462.022241:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768462.022245:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768462.022246:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768462.022250:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5083 00000100:00080000:1.0:1775768462.022253:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768462.022256:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768462.022265:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768462.022289:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967424640:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775768462.022293:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e7b80 x1862022967424768/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768462.022313:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768462.022317:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768462.022318:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768462.022322:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5083 00000100:00080000:1.0:1775768462.022326:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768462.022330:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768462.022334:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768462.022345:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e4e00 x1862022967424896/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768462.022353:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1775768462.022355:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:1.0:1775768462.022359:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768462.022361:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768462.022365:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5083 00000100:00080000:1.0:1775768462.022369:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768462.022373:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:1.0:1775768462.022378:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768462.022409:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768462/real 1775768462] req@ffff8f994f2e4700 x1862022967424640/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768462.022429:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4700 x1862022967424640/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768462.022434:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f994f2e6300 x1862022967425024/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768462.022453:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768462.022459:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768462.022464:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967424640:192.168.202.146@tcp:250:kworker.0 00000100:00100000:3.0:1775768462.022507:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967424768:192.168.202.146@tcp:38:kworker.0 00000800:00000100:3.0:1775768462.022530:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768462.022606:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768462/real 1775768462] req@ffff8f994f2e7b80 x1862022967424768/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768462.022620:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e7b80 x1862022967424768/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768462.022627:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768462.022630:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768462.022633:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967424768:192.168.202.146@tcp:38:kworker.0 00000100:00100000:3.0:1775768462.022639:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967424896:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768462.022660:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768462.022670:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768462/real 1775768462] req@ffff8f994f2e4e00 x1862022967424896/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768462.022681:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e4e00 x1862022967424896/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768462.022688:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768462.022691:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768462.022694:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967424896:192.168.202.146@tcp:8:kworker.0 00000100:00100000:3.0:1775768462.022700:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967425024:192.168.202.146@tcp:8:kworker.0 00000800:00000100:3.0:1775768462.022719:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:3.0:1775768462.022729:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768462/real 1775768462] req@ffff8f994f2e6300 x1862022967425024/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768462.022797:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f994f2e6300 x1862022967425024/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768462.022805:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768462.022809:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:3.0:1775768462.022812:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f994f2e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967425024:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768462.917874:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768462.917949:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768463.941844:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768463.941925:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768464.965960:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768464.965989:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775768465.989857:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768465.989941:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768467.013920:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768467.013926:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768467.141934:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768467.141945:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768467.141949:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768467.141960:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 5088 00000100:00080000:3.0:1775768467.141971:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768467.141978:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:3.0:1775768467.142018:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970ede680 x1862022967425152/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768467.142085:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967425152:192.168.202.146@tcp:250:kworker.0 00000800:00000100:1.0:1775768467.142187:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00080000:3.0:1775768467.144603:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768467.144611:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768467.144615:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768467.144624:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5088 00000100:00080000:3.0:1775768467.144633:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768467.144640:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768467.144652:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768467.144684:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970eded80 x1862022967425280/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768467.145878:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768467.145884:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768467.145887:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768467.145894:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5088 00000100:00080000:3.0:1775768467.145901:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768467.145906:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768467.145933:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768467.145962:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf480 x1862022967425408/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768467.146173:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768467/real 1775768467] req@ffff8f9970ede680 x1862022967425152/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768467.146226:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970ede680 x1862022967425152/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768467.146558:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768467.146562:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768467.146564:0:36:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768467.146569:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5088 00000100:00080000:3.0:1775768467.146592:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768467.146597:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:3.0:1775768467.146603:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768467.146617:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970edf800 x1862022967425536/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768467.146657:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768467.146661:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768467.146665:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970ede680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967425152:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775768467.146698:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967425280:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775768467.146853:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768467.146874:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768467/real 1775768467] req@ffff8f9970eded80 x1862022967425280/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768467.146888:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970eded80 x1862022967425280/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768467.146896:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768467.146899:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768467.146902:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970eded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967425280:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775768467.146908:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967425408:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775768467.146931:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768467.146942:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768467/real 1775768467] req@ffff8f9970edf480 x1862022967425408/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768467.146991:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf480 x1862022967425408/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768467.146999:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768467.147002:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768467.147006:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967425408:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775768467.147011:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967425536:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775768467.147029:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768467.147038:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768467/real 1775768467] req@ffff8f9970edf800 x1862022967425536/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768467.147048:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970edf800 x1862022967425536/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768467.147055:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768467.147058:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768467.147061:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970edf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967425536:192.168.202.146@tcp:8:kworker.0 00010000:00010000:0.0:1775768468.038181:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768468.038203:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775768469.061844:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768469.061946:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768470.085875:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768470.086015:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768471.109902:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768471.110374:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00000100:00100000:2.0:1775768472.133855:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768472.134556:0:19801:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768472.265221:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 884ae1b0-72f8-4a88-a129-db99cded7efc->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768472.265232:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768472.265237:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768472.265246:0:24799:0:(import.c:509:import_select_connection()) MGC192.168.202.146@tcp: connect to NID 192.168.202.146@tcp last attempt 5093 00000100:00080000:0.0:1775768472.265258:0:24799:0:(import.c:588:import_select_connection()) MGC192.168.202.146@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768472.265264:0:24799:0:(import.c:623:import_select_connection()) MGC192.168.202.146@tcp: import ffff8f9952315800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000100:00100000:0.0:1775768472.265299:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74700 x1862022967425664/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768472.265333:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768472.265336:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768472.265338:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768472.265343:0:24799:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5093 00000100:00080000:0.0:1775768472.265346:0:24799:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768472.265350:0:24799:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8f99523ee800: import ffff8f9952a6b000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768472.265359:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768472.265372:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a76d80 x1862022967425792/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768472.265382:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775768472.265385:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967425664:192.168.202.146@tcp:250:kworker.0 00000100:00080000:0.0:1775768472.265386:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768472.265390:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768472.265396:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5093 00000100:00080000:0.0:1775768472.265399:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768472.265402:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8f99523ee800: import ffff8f9952a6d800 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000008:00100000:0.0:1775768472.265407:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768472.265417:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a77800 x1862022967425920/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768472.265430:0:24799:0:(pinger.c:229:ptlrpc_pinger_process_import()) 3f83546c-01a9-4156-8a97-44c446245d3e->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768472.265433:0:24799:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775768472.265434:0:24799:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768472.265437:0:24799:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: connect to NID 192.168.202.146@tcp last attempt 5093 00000100:00080000:0.0:1775768472.265447:0:24799:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768472.265450:0:24799:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8f99523ee800: import ffff8f9952a6f000 using connection 192.168.202.146@tcp/192.168.202.146@tcp 00000800:00000100:1.0:1775768472.265454:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000008:00100000:0.0:1775768472.265455:0:24799:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768472.265467:0:24799:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8f9970a74380 x1862022967426048/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775768472.265500:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768472/real 1775768472] req@ffff8f9970a74700 x1862022967425664/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768472.265525:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74700 x1862022967425664/t0(0) o250->MGC192.168.202.146@tcp@192.168.202.146@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768472.265534:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952315800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768472.265538:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768472.265542:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:884ae1b0-72f8-4a88-a129-db99cded7efc:2378:1862022967425664:192.168.202.146@tcp:250:kworker.0 00000100:00100000:1.0:1775768472.265564:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967425792:192.168.202.146@tcp:38:kworker.0 00000800:00000100:1.0:1775768472.265586:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768472.265598:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768472/real 1775768472] req@ffff8f9970a76d80 x1862022967425792/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768472.265610:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a76d80 x1862022967425792/t0(0) o38->lustre-MDT0000-mdc-ffff8f99523ee800@192.168.202.146@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768472.265618:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6b000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768472.265620:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768472.265623:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a76d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967425792:192.168.202.146@tcp:38:kworker.0 00000100:00100000:1.0:1775768472.265629:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967425920:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775768472.265730:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768472.265860:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768472/real 1775768472] req@ffff8f9970a77800 x1862022967425920/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768472.265875:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a77800 x1862022967425920/t0(0) o8->lustre-OST0000-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768472.265888:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6d800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768472.265891:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768472.265894:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a77800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967425920:192.168.202.146@tcp:8:kworker.0 00000100:00100000:1.0:1775768472.265900:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967426048:192.168.202.146@tcp:8:kworker.0 00000800:00000100:1.0:1775768472.265922:0:2378:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.146@tcp 00000100:00100000:1.0:1775768472.265933:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768472/real 1775768472] req@ffff8f9970a74380 x1862022967426048/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768472.265944:0:2378:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8f9970a74380 x1862022967426048/t0(0) o8->lustre-OST0001-osc-ffff8f99523ee800@192.168.202.146@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768472.265952:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8f9952a6f000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768472.265955:0:2378:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.146@tcp failed (-110) 00000100:00100000:1.0:1775768472.265958:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8f9970a74380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:3f83546c-01a9-4156-8a97-44c446245d3e:2378:1862022967426048:192.168.202.146@tcp:8:kworker.0 00000100:00100000:2.0:1775768473.157813:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:0.0:1775768473.157957:0:11138:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768474.181914:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768474.189253:0:5571:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768475.205842:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768475.206121:0:11210:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775768476.230080:0:5572:0:(client.c:2691:ptlrpc_set_wait()) set ffff8f99703af840 going to sleep for 0 seconds 00010000:00010000:1.0:1775768476.230395:0:19800:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) Debug log: 74870 lines, 74870 kept, 0 dropped, 0 bad.